2024-11-20 17:04:34,679 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-20 17:04:34,695 main DEBUG Took 0.012930 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 17:04:34,695 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 17:04:34,696 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 17:04:34,697 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 17:04:34,698 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,707 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 17:04:34,724 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,727 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,728 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,729 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,731 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,732 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,734 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,736 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,737 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,738 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,739 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,740 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,742 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:04:34,743 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,743 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 17:04:34,745 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:04:34,747 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 17:04:34,749 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 17:04:34,750 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 17:04:34,752 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 17:04:34,752 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 17:04:34,763 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 17:04:34,767 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 17:04:34,769 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 17:04:34,769 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 17:04:34,770 main DEBUG createAppenders(={Console}) 2024-11-20 17:04:34,771 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-20 17:04:34,771 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-20 17:04:34,772 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-20 17:04:34,772 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 17:04:34,773 main DEBUG OutputStream closed 2024-11-20 17:04:34,773 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 17:04:34,774 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 17:04:34,774 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-20 17:04:34,858 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 17:04:34,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 17:04:34,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 17:04:34,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 17:04:34,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 17:04:34,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 17:04:34,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 17:04:34,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 17:04:34,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 17:04:34,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 17:04:34,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 17:04:34,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 17:04:34,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 17:04:34,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 17:04:34,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 17:04:34,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 17:04:34,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 17:04:34,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 17:04:34,871 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 17:04:34,872 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-20 17:04:34,873 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 17:04:34,874 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-20T17:04:34,892 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-20 17:04:34,896 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 17:04:34,896 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T17:04:35,151 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3 2024-11-20T17:04:35,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T17:04:35,184 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9, deleteOnExit=true 2024-11-20T17:04:35,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T17:04:35,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/test.cache.data in system properties and HBase conf 2024-11-20T17:04:35,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T17:04:35,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/hadoop.log.dir in system properties and HBase conf 2024-11-20T17:04:35,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T17:04:35,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T17:04:35,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T17:04:35,278 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T17:04:35,362 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:04:35,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T17:04:35,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T17:04:35,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T17:04:35,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T17:04:35,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T17:04:35,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T17:04:35,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T17:04:35,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T17:04:35,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T17:04:35,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/nfs.dump.dir in system properties and HBase conf 2024-11-20T17:04:35,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/java.io.tmpdir in system properties and HBase conf 2024-11-20T17:04:35,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T17:04:35,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T17:04:35,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T17:04:36,349 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T17:04:36,414 INFO [Time-limited test {}] log.Log(170): Logging initialized @2388ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T17:04:36,479 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:04:36,534 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T17:04:36,554 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T17:04:36,554 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T17:04:36,555 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T17:04:36,566 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:04:36,569 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/hadoop.log.dir/,AVAILABLE} 2024-11-20T17:04:36,570 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T17:04:36,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/java.io.tmpdir/jetty-localhost-40277-hadoop-hdfs-3_4_1-tests_jar-_-any-11135504622187389868/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T17:04:36,747 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:40277} 2024-11-20T17:04:36,747 INFO [Time-limited test {}] server.Server(415): Started @2721ms 2024-11-20T17:04:37,249 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:04:37,259 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T17:04:37,260 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T17:04:37,260 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T17:04:37,260 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T17:04:37,261 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/hadoop.log.dir/,AVAILABLE} 2024-11-20T17:04:37,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T17:04:37,362 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/java.io.tmpdir/jetty-localhost-45565-hadoop-hdfs-3_4_1-tests_jar-_-any-12380720673747766984/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T17:04:37,363 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:45565} 2024-11-20T17:04:37,363 INFO [Time-limited test {}] server.Server(415): Started @3338ms 2024-11-20T17:04:37,412 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T17:04:37,519 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:04:37,524 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T17:04:37,526 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T17:04:37,526 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T17:04:37,526 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T17:04:37,531 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/hadoop.log.dir/,AVAILABLE} 2024-11-20T17:04:37,532 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T17:04:37,631 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/java.io.tmpdir/jetty-localhost-45221-hadoop-hdfs-3_4_1-tests_jar-_-any-13465521643027839112/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T17:04:37,632 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:45221} 2024-11-20T17:04:37,632 INFO [Time-limited test {}] server.Server(415): Started @3607ms 2024-11-20T17:04:37,634 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T17:04:37,671 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:04:37,677 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T17:04:37,683 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T17:04:37,683 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T17:04:37,684 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T17:04:37,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/hadoop.log.dir/,AVAILABLE} 2024-11-20T17:04:37,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T17:04:37,793 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/java.io.tmpdir/jetty-localhost-38555-hadoop-hdfs-3_4_1-tests_jar-_-any-12338593769043570715/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T17:04:37,794 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:38555} 2024-11-20T17:04:37,794 INFO [Time-limited test {}] server.Server(415): Started @3768ms 2024-11-20T17:04:37,796 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T17:04:38,657 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/data/data2/current/BP-534784878-172.17.0.2-1732122275921/current, will proceed with Du for space computation calculation, 2024-11-20T17:04:38,657 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/data/data4/current/BP-534784878-172.17.0.2-1732122275921/current, will proceed with Du for space computation calculation, 2024-11-20T17:04:38,657 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/data/data3/current/BP-534784878-172.17.0.2-1732122275921/current, will proceed with Du for space computation calculation, 2024-11-20T17:04:38,657 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/data/data1/current/BP-534784878-172.17.0.2-1732122275921/current, will proceed with Du for space computation calculation, 2024-11-20T17:04:38,688 WARN [Thread-136 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/data/data5/current/BP-534784878-172.17.0.2-1732122275921/current, will proceed with Du for space computation calculation, 2024-11-20T17:04:38,689 WARN [Thread-137 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/data/data6/current/BP-534784878-172.17.0.2-1732122275921/current, will proceed with Du for space computation calculation, 2024-11-20T17:04:38,690 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T17:04:38,691 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T17:04:38,713 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T17:04:38,740 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6acf98db5dd5d09 with lease ID 0x158fbbb8055a7561: Processing first storage report for DS-7004819e-1508-46b4-86e4-acb222d36d37 from datanode DatanodeRegistration(127.0.0.1:33493, datanodeUuid=706c2a1a-c5f1-48cd-b8d8-e3857f164abc, infoPort=33029, infoSecurePort=0, ipcPort=43381, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921) 2024-11-20T17:04:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6acf98db5dd5d09 with lease ID 0x158fbbb8055a7561: from storage DS-7004819e-1508-46b4-86e4-acb222d36d37 node DatanodeRegistration(127.0.0.1:33493, datanodeUuid=706c2a1a-c5f1-48cd-b8d8-e3857f164abc, infoPort=33029, infoSecurePort=0, ipcPort=43381, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T17:04:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3b9e18fc0245ceac with lease ID 0x158fbbb8055a7560: Processing first storage report for DS-db2b7c12-a09c-4d0d-aea8-9fea2ae5a604 from datanode DatanodeRegistration(127.0.0.1:41999, datanodeUuid=5879d815-8426-45b5-a1bc-b72dbf9bcb74, infoPort=45507, infoSecurePort=0, ipcPort=41021, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921) 2024-11-20T17:04:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3b9e18fc0245ceac with lease ID 0x158fbbb8055a7560: from storage DS-db2b7c12-a09c-4d0d-aea8-9fea2ae5a604 node DatanodeRegistration(127.0.0.1:41999, datanodeUuid=5879d815-8426-45b5-a1bc-b72dbf9bcb74, infoPort=45507, infoSecurePort=0, ipcPort=41021, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T17:04:38,742 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4e3936a4588841fd with lease ID 0x158fbbb8055a7562: Processing first storage report for DS-4e1882e0-2569-4b10-92a0-f5eec9f95d96 from datanode DatanodeRegistration(127.0.0.1:33541, datanodeUuid=fa4da56d-3288-42f7-b396-c2214edac1a2, infoPort=37891, infoSecurePort=0, ipcPort=39159, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921) 2024-11-20T17:04:38,742 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4e3936a4588841fd with lease ID 0x158fbbb8055a7562: from storage DS-4e1882e0-2569-4b10-92a0-f5eec9f95d96 node DatanodeRegistration(127.0.0.1:33541, datanodeUuid=fa4da56d-3288-42f7-b396-c2214edac1a2, infoPort=37891, infoSecurePort=0, ipcPort=39159, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T17:04:38,742 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6acf98db5dd5d09 with lease ID 0x158fbbb8055a7561: Processing first storage report for DS-2328dc68-f49e-457c-9a6b-b99f6c84c777 from datanode DatanodeRegistration(127.0.0.1:33493, datanodeUuid=706c2a1a-c5f1-48cd-b8d8-e3857f164abc, infoPort=33029, infoSecurePort=0, ipcPort=43381, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921) 2024-11-20T17:04:38,742 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6acf98db5dd5d09 with lease ID 0x158fbbb8055a7561: from storage DS-2328dc68-f49e-457c-9a6b-b99f6c84c777 node DatanodeRegistration(127.0.0.1:33493, datanodeUuid=706c2a1a-c5f1-48cd-b8d8-e3857f164abc, infoPort=33029, infoSecurePort=0, ipcPort=43381, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T17:04:38,742 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3b9e18fc0245ceac with lease ID 0x158fbbb8055a7560: Processing first storage report for DS-824bbaf3-a2e2-4adc-83b3-fef3b4b00851 from datanode DatanodeRegistration(127.0.0.1:41999, datanodeUuid=5879d815-8426-45b5-a1bc-b72dbf9bcb74, infoPort=45507, infoSecurePort=0, ipcPort=41021, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921) 2024-11-20T17:04:38,743 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3b9e18fc0245ceac with lease ID 0x158fbbb8055a7560: from storage DS-824bbaf3-a2e2-4adc-83b3-fef3b4b00851 node DatanodeRegistration(127.0.0.1:41999, datanodeUuid=5879d815-8426-45b5-a1bc-b72dbf9bcb74, infoPort=45507, infoSecurePort=0, ipcPort=41021, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T17:04:38,743 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4e3936a4588841fd with lease ID 0x158fbbb8055a7562: Processing first storage report for DS-c569a37c-b775-4d92-885a-141babfa41fa from datanode DatanodeRegistration(127.0.0.1:33541, datanodeUuid=fa4da56d-3288-42f7-b396-c2214edac1a2, infoPort=37891, infoSecurePort=0, ipcPort=39159, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921) 2024-11-20T17:04:38,743 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4e3936a4588841fd with lease ID 0x158fbbb8055a7562: from storage DS-c569a37c-b775-4d92-885a-141babfa41fa node DatanodeRegistration(127.0.0.1:33541, datanodeUuid=fa4da56d-3288-42f7-b396-c2214edac1a2, infoPort=37891, infoSecurePort=0, ipcPort=39159, storageInfo=lv=-57;cid=testClusterID;nsid=952777985;c=1732122275921), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T17:04:38,790 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3 2024-11-20T17:04:38,852 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/zookeeper_0, clientPort=57068, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/587f30de-ddfd-22ba-2cc3-86b9420785f3/cluster_297672a8-4db3-366a-28a4-52c68b28efa9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T17:04:38,862 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57068 2024-11-20T17:04:38,871 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:38,874 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:39,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741825_1001 (size=7) 2024-11-20T17:04:39,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741825_1001 (size=7) 2024-11-20T17:04:39,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741825_1001 (size=7) 2024-11-20T17:04:39,495 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e with version=8 2024-11-20T17:04:39,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/hbase-staging 2024-11-20T17:04:39,566 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T17:04:39,800 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b60eac9bda43:0 server-side Connection retries=45 2024-11-20T17:04:39,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:39,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:39,813 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:04:39,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:39,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:04:39,944 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T17:04:40,004 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T17:04:40,014 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T17:04:40,017 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:04:40,044 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 106442 (auto-detected) 2024-11-20T17:04:40,045 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T17:04:40,065 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44751 2024-11-20T17:04:40,089 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44751 connecting to ZooKeeper ensemble=127.0.0.1:57068 2024-11-20T17:04:40,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:447510x0, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:04:40,193 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44751-0x1015a7e0e7a0000 connected 2024-11-20T17:04:40,268 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,273 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,285 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:04:40,289 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e, hbase.cluster.distributed=false 2024-11-20T17:04:40,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:04:40,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44751 2024-11-20T17:04:40,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44751 2024-11-20T17:04:40,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44751 2024-11-20T17:04:40,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44751 2024-11-20T17:04:40,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44751 2024-11-20T17:04:40,413 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b60eac9bda43:0 server-side Connection retries=45 2024-11-20T17:04:40,415 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,415 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,415 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:04:40,415 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,415 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:04:40,418 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:04:40,421 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:04:40,422 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34957 2024-11-20T17:04:40,424 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34957 connecting to ZooKeeper ensemble=127.0.0.1:57068 2024-11-20T17:04:40,425 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349570x0, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:04:40,449 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34957-0x1015a7e0e7a0001 connected 2024-11-20T17:04:40,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:349570x0, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:04:40,453 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:04:40,461 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:04:40,464 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:04:40,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:04:40,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34957 2024-11-20T17:04:40,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34957 2024-11-20T17:04:40,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34957 2024-11-20T17:04:40,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34957 2024-11-20T17:04:40,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34957 2024-11-20T17:04:40,489 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b60eac9bda43:0 server-side Connection retries=45 2024-11-20T17:04:40,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,490 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:04:40,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:04:40,490 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:04:40,490 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:04:40,491 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36645 2024-11-20T17:04:40,493 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36645 connecting to ZooKeeper ensemble=127.0.0.1:57068 2024-11-20T17:04:40,494 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,497 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366450x0, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:04:40,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:366450x0, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:04:40,533 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36645-0x1015a7e0e7a0002 connected 2024-11-20T17:04:40,533 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:04:40,534 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:04:40,535 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:04:40,537 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:04:40,537 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36645 2024-11-20T17:04:40,537 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36645 2024-11-20T17:04:40,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36645 2024-11-20T17:04:40,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36645 2024-11-20T17:04:40,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36645 2024-11-20T17:04:40,557 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b60eac9bda43:0 server-side Connection retries=45 2024-11-20T17:04:40,557 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,557 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,557 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:04:40,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:04:40,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:04:40,558 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:04:40,558 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:04:40,559 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36999 2024-11-20T17:04:40,561 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36999 connecting to ZooKeeper ensemble=127.0.0.1:57068 2024-11-20T17:04:40,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,564 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369990x0, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:04:40,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:369990x0, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:04:40,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36999-0x1015a7e0e7a0003 connected 2024-11-20T17:04:40,575 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:04:40,578 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:04:40,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:04:40,581 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:04:40,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36999 2024-11-20T17:04:40,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36999 2024-11-20T17:04:40,586 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36999 2024-11-20T17:04:40,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36999 2024-11-20T17:04:40,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36999 2024-11-20T17:04:40,600 DEBUG [M:0;b60eac9bda43:44751 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b60eac9bda43:44751 2024-11-20T17:04:40,601 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b60eac9bda43,44751,1732122279653 2024-11-20T17:04:40,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:04:40,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:04:40,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:04:40,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:04:40,618 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b60eac9bda43,44751,1732122279653 2024-11-20T17:04:40,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:04:40,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:04:40,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:04:40,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:40,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:40,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:40,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:40,650 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T17:04:40,651 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b60eac9bda43,44751,1732122279653 from backup master directory 2024-11-20T17:04:40,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:04:40,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b60eac9bda43,44751,1732122279653 2024-11-20T17:04:40,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:04:40,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:04:40,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:04:40,658 WARN [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:04:40,658 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b60eac9bda43,44751,1732122279653 2024-11-20T17:04:40,660 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T17:04:40,661 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T17:04:40,712 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/hbase.id] with ID: a3098b73-f92a-4174-8ea2-e2512d71554a 2024-11-20T17:04:40,712 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/.tmp/hbase.id 2024-11-20T17:04:40,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741826_1002 (size=42) 2024-11-20T17:04:40,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741826_1002 (size=42) 2024-11-20T17:04:40,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741826_1002 (size=42) 2024-11-20T17:04:40,729 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/.tmp/hbase.id]:[hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/hbase.id] 2024-11-20T17:04:40,775 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:04:40,780 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T17:04:40,797 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-20T17:04:40,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:40,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:40,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:40,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:40,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741827_1003 (size=196) 2024-11-20T17:04:40,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741827_1003 (size=196) 2024-11-20T17:04:40,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741827_1003 (size=196) 2024-11-20T17:04:40,853 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:04:40,854 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T17:04:40,867 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:04:40,871 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:04:40,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741828_1004 (size=1189) 2024-11-20T17:04:40,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741828_1004 (size=1189) 2024-11-20T17:04:40,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741828_1004 (size=1189) 2024-11-20T17:04:40,916 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/data/master/store 2024-11-20T17:04:40,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741829_1005 (size=34) 2024-11-20T17:04:40,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741829_1005 (size=34) 2024-11-20T17:04:40,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741829_1005 (size=34) 2024-11-20T17:04:40,942 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T17:04:40,946 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:40,947 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:04:40,947 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:04:40,947 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:04:40,949 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:04:40,949 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:04:40,949 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:04:40,950 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732122280947Disabling compacts and flushes for region at 1732122280947Disabling writes for close at 1732122280949 (+2 ms)Writing region close event to WAL at 1732122280949Closed at 1732122280949 2024-11-20T17:04:40,952 WARN [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/data/master/store/.initializing 2024-11-20T17:04:40,953 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/WALs/b60eac9bda43,44751,1732122279653 2024-11-20T17:04:40,961 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:04:40,974 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C44751%2C1732122279653, suffix=, logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/WALs/b60eac9bda43,44751,1732122279653, archiveDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/oldWALs, maxLogs=10 2024-11-20T17:04:40,995 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/WALs/b60eac9bda43,44751,1732122279653/b60eac9bda43%2C44751%2C1732122279653.1732122280978, exclude list is [], retry=0 2024-11-20T17:04:41,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-4e1882e0-2569-4b10-92a0-f5eec9f95d96,DISK] 2024-11-20T17:04:41,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41999,DS-db2b7c12-a09c-4d0d-aea8-9fea2ae5a604,DISK] 2024-11-20T17:04:41,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33493,DS-7004819e-1508-46b4-86e4-acb222d36d37,DISK] 2024-11-20T17:04:41,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T17:04:41,058 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/WALs/b60eac9bda43,44751,1732122279653/b60eac9bda43%2C44751%2C1732122279653.1732122280978 2024-11-20T17:04:41,059 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37891:37891),(127.0.0.1/127.0.0.1:45507:45507),(127.0.0.1/127.0.0.1:33029:33029)] 2024-11-20T17:04:41,060 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:04:41,060 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:41,063 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,064 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T17:04:41,130 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:41,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,137 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T17:04:41,137 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:41,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T17:04:41,143 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:41,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,146 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T17:04:41,147 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:41,148 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,151 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,153 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,158 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,158 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,161 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:04:41,165 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:04:41,170 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:41,171 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63036714, jitterRate=-0.06067976355552673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:04:41,180 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732122281076Initializing all the Stores at 1732122281079 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122281079Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122281080 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122281080Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122281080Cleaning up temporary data from old regions at 1732122281158 (+78 ms)Region opened successfully at 1732122281180 (+22 ms) 2024-11-20T17:04:41,181 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T17:04:41,215 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@660b8c4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b60eac9bda43/172.17.0.2:0 2024-11-20T17:04:41,245 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T17:04:41,254 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:41,254 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:41,257 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:41,258 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T17:04:41,264 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T17:04:41,264 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T17:04:41,287 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T17:04:41,294 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T17:04:41,332 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T17:04:41,335 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T17:04:41,338 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T17:04:41,348 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T17:04:41,351 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T17:04:41,355 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T17:04:41,365 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T17:04:41,366 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T17:04:41,373 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T17:04:41,392 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T17:04:41,398 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T17:04:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:04:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:04:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:04:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:04:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,412 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b60eac9bda43,44751,1732122279653, sessionid=0x1015a7e0e7a0000, setting cluster-up flag (Was=false) 2024-11-20T17:04:41,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,465 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T17:04:41,468 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b60eac9bda43,44751,1732122279653 2024-11-20T17:04:41,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:41,515 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T17:04:41,517 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b60eac9bda43,44751,1732122279653 2024-11-20T17:04:41,527 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T17:04:41,591 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(746): ClusterId : a3098b73-f92a-4174-8ea2-e2512d71554a 2024-11-20T17:04:41,591 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(746): ClusterId : a3098b73-f92a-4174-8ea2-e2512d71554a 2024-11-20T17:04:41,591 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(746): ClusterId : a3098b73-f92a-4174-8ea2-e2512d71554a 2024-11-20T17:04:41,594 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:04:41,594 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:04:41,594 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:04:41,594 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:41,603 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T17:04:41,609 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:04:41,609 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:04:41,609 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:04:41,609 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:04:41,609 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:04:41,609 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:04:41,610 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T17:04:41,616 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:04:41,616 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:04:41,616 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:04:41,617 DEBUG [RS:2;b60eac9bda43:36999 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371495ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b60eac9bda43/172.17.0.2:0 2024-11-20T17:04:41,617 DEBUG [RS:1;b60eac9bda43:36645 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d292cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b60eac9bda43/172.17.0.2:0 2024-11-20T17:04:41,617 DEBUG [RS:0;b60eac9bda43:34957 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c8ed18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b60eac9bda43/172.17.0.2:0 2024-11-20T17:04:41,617 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b60eac9bda43,44751,1732122279653 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T17:04:41,633 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b60eac9bda43:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:04:41,634 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b60eac9bda43:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:04:41,634 DEBUG [RS:1;b60eac9bda43:36645 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b60eac9bda43:36645 2024-11-20T17:04:41,634 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b60eac9bda43:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:04:41,634 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b60eac9bda43:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:04:41,634 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b60eac9bda43:0, corePoolSize=10, maxPoolSize=10 2024-11-20T17:04:41,635 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,635 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b60eac9bda43:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:04:41,635 DEBUG [RS:2;b60eac9bda43:36999 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b60eac9bda43:36999 2024-11-20T17:04:41,635 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,638 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:04:41,638 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:04:41,638 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:04:41,638 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:04:41,638 DEBUG [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:04:41,638 DEBUG [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:04:41,639 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122311639 2024-11-20T17:04:41,641 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T17:04:41,642 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(2659): reportForDuty to master=b60eac9bda43,44751,1732122279653 with port=36645, startcode=1732122280488 2024-11-20T17:04:41,642 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T17:04:41,644 DEBUG [RS:0;b60eac9bda43:34957 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b60eac9bda43:34957 2024-11-20T17:04:41,644 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:04:41,644 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:04:41,644 DEBUG [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:04:41,645 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(2659): reportForDuty to master=b60eac9bda43,44751,1732122279653 with port=34957, startcode=1732122280382 2024-11-20T17:04:41,646 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:41,645 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(2659): reportForDuty to master=b60eac9bda43,44751,1732122279653 with port=36999, startcode=1732122280556 2024-11-20T17:04:41,646 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T17:04:41,647 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T17:04:41,647 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T17:04:41,648 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T17:04:41,648 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T17:04:41,650 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,653 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,654 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T17:04:41,659 DEBUG [RS:2;b60eac9bda43:36999 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:04:41,659 DEBUG [RS:1;b60eac9bda43:36645 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:04:41,659 DEBUG [RS:0;b60eac9bda43:34957 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:04:41,668 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T17:04:41,670 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T17:04:41,670 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T17:04:41,675 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T17:04:41,675 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T17:04:41,680 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.large.0-1732122281677,5,FailOnTimeoutGroup] 2024-11-20T17:04:41,690 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.small.0-1732122281681,5,FailOnTimeoutGroup] 2024-11-20T17:04:41,690 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,691 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T17:04:41,724 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,724 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741831_1007 (size=1321) 2024-11-20T17:04:41,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741831_1007 (size=1321) 2024-11-20T17:04:41,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741831_1007 (size=1321) 2024-11-20T17:04:41,730 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T17:04:41,731 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:41,735 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58153, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:04:41,735 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44671, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:04:41,735 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46409, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:04:41,741 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b60eac9bda43,34957,1732122280382 2024-11-20T17:04:41,744 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] master.ServerManager(517): Registering regionserver=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:41,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741832_1008 (size=32) 2024-11-20T17:04:41,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741832_1008 (size=32) 2024-11-20T17:04:41,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741832_1008 (size=32) 2024-11-20T17:04:41,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:41,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T17:04:41,756 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b60eac9bda43,36999,1732122280556 2024-11-20T17:04:41,756 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] master.ServerManager(517): Registering regionserver=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:41,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:04:41,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:41,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T17:04:41,760 DEBUG [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:41,760 DEBUG [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34631 2024-11-20T17:04:41,760 DEBUG [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:04:41,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:04:41,764 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,764 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b60eac9bda43,36645,1732122280488 2024-11-20T17:04:41,764 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] master.ServerManager(517): Registering regionserver=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:41,764 DEBUG [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:41,764 DEBUG [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34631 2024-11-20T17:04:41,764 DEBUG [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:04:41,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:41,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T17:04:41,768 DEBUG [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:41,768 DEBUG [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34631 2024-11-20T17:04:41,769 DEBUG [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:04:41,769 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:04:41,769 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:41,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T17:04:41,774 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:04:41,774 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:41,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:41,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:04:41,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740 2024-11-20T17:04:41,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740 2024-11-20T17:04:41,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:04:41,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:04:41,781 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:04:41,784 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:04:41,788 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:41,789 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71038407, jitterRate=0.0585547536611557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:04:41,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:04:41,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732122281752Initializing all the Stores at 1732122281754 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122281754Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122281754Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122281754Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122281754Cleaning up temporary data from old regions at 1732122281780 (+26 ms)Region opened successfully at 1732122281795 (+15 ms) 2024-11-20T17:04:41,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:04:41,795 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:04:41,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:04:41,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:04:41,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:04:41,797 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:04:41,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732122281795Disabling compacts and flushes for region at 1732122281795Disabling writes for close at 1732122281795Writing region close event to WAL at 1732122281796 (+1 ms)Closed at 1732122281797 (+1 ms) 2024-11-20T17:04:41,800 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:41,800 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T17:04:41,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T17:04:41,817 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:41,819 DEBUG [RS:0;b60eac9bda43:34957 {}] zookeeper.ZKUtil(111): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b60eac9bda43,34957,1732122280382 2024-11-20T17:04:41,819 WARN [RS:0;b60eac9bda43:34957 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:04:41,819 DEBUG [RS:1;b60eac9bda43:36645 {}] zookeeper.ZKUtil(111): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b60eac9bda43,36645,1732122280488 2024-11-20T17:04:41,819 DEBUG [RS:2;b60eac9bda43:36999 {}] zookeeper.ZKUtil(111): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b60eac9bda43,36999,1732122280556 2024-11-20T17:04:41,819 INFO [RS:0;b60eac9bda43:34957 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:04:41,819 WARN [RS:1;b60eac9bda43:36645 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:04:41,819 WARN [RS:2;b60eac9bda43:36999 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:04:41,820 INFO [RS:2;b60eac9bda43:36999 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:04:41,820 INFO [RS:1;b60eac9bda43:36645 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:04:41,820 DEBUG [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382 2024-11-20T17:04:41,820 DEBUG [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36999,1732122280556 2024-11-20T17:04:41,820 DEBUG [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36645,1732122280488 2024-11-20T17:04:41,820 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T17:04:41,821 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b60eac9bda43,36999,1732122280556] 2024-11-20T17:04:41,821 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b60eac9bda43,36645,1732122280488] 2024-11-20T17:04:41,821 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b60eac9bda43,34957,1732122280382] 2024-11-20T17:04:41,849 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:04:41,849 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:04:41,849 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:04:41,870 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:04:41,870 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:04:41,870 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:04:41,875 INFO [RS:0;b60eac9bda43:34957 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:04:41,875 INFO [RS:1;b60eac9bda43:36645 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:04:41,875 INFO [RS:2;b60eac9bda43:36999 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:04:41,876 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,876 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,876 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,877 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:04:41,877 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:04:41,878 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:04:41,883 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:04:41,883 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:04:41,885 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,885 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,885 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,885 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,885 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:04:41,885 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,885 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,885 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,885 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,885 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,885 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,885 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,885 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:04:41,886 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:04:41,886 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:04:41,886 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,886 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b60eac9bda43:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:04:41,887 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b60eac9bda43:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:04:41,887 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b60eac9bda43:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:04:41,887 DEBUG [RS:0;b60eac9bda43:34957 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:04:41,887 DEBUG [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:04:41,887 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b60eac9bda43:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:04:41,887 DEBUG [RS:2;b60eac9bda43:36999 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b60eac9bda43:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:04:41,890 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,891 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,891 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,891 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,891 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,891 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,36999,1732122280556-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:04:41,894 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,34957,1732122280382-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,895 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,36645,1732122280488-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:04:41,915 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:04:41,915 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:04:41,915 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:04:41,917 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,36645,1732122280488-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,917 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,36999,1732122280556-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,917 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,34957,1732122280382-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,917 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,917 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,917 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,917 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.Replication(171): b60eac9bda43,36645,1732122280488 started 2024-11-20T17:04:41,917 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.Replication(171): b60eac9bda43,36999,1732122280556 started 2024-11-20T17:04:41,917 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.Replication(171): b60eac9bda43,34957,1732122280382 started 2024-11-20T17:04:41,934 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,935 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,935 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1482): Serving as b60eac9bda43,36645,1732122280488, RpcServer on b60eac9bda43/172.17.0.2:36645, sessionid=0x1015a7e0e7a0002 2024-11-20T17:04:41,935 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(1482): Serving as b60eac9bda43,36999,1732122280556, RpcServer on b60eac9bda43/172.17.0.2:36999, sessionid=0x1015a7e0e7a0003 2024-11-20T17:04:41,935 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:04:41,935 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:04:41,936 DEBUG [RS:2;b60eac9bda43:36999 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b60eac9bda43,36999,1732122280556 2024-11-20T17:04:41,936 DEBUG [RS:1;b60eac9bda43:36645 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b60eac9bda43,36645,1732122280488 2024-11-20T17:04:41,936 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b60eac9bda43,36999,1732122280556' 2024-11-20T17:04:41,936 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b60eac9bda43,36645,1732122280488' 2024-11-20T17:04:41,936 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:04:41,936 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:04:41,937 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:04:41,937 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:04:41,937 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:04:41,937 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:04:41,937 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:04:41,937 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:04:41,938 DEBUG [RS:2;b60eac9bda43:36999 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b60eac9bda43,36999,1732122280556 2024-11-20T17:04:41,938 DEBUG [RS:1;b60eac9bda43:36645 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b60eac9bda43,36645,1732122280488 2024-11-20T17:04:41,938 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b60eac9bda43,36999,1732122280556' 2024-11-20T17:04:41,938 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b60eac9bda43,36645,1732122280488' 2024-11-20T17:04:41,938 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:04:41,938 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:04:41,938 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:04:41,938 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:04:41,939 DEBUG [RS:2;b60eac9bda43:36999 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:04:41,939 DEBUG [RS:1;b60eac9bda43:36645 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:04:41,939 INFO [RS:2;b60eac9bda43:36999 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:04:41,939 INFO [RS:1;b60eac9bda43:36645 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:04:41,939 INFO [RS:2;b60eac9bda43:36999 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:04:41,939 INFO [RS:1;b60eac9bda43:36645 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:04:41,939 INFO [RS:0;b60eac9bda43:34957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:41,940 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(1482): Serving as b60eac9bda43,34957,1732122280382, RpcServer on b60eac9bda43/172.17.0.2:34957, sessionid=0x1015a7e0e7a0001 2024-11-20T17:04:41,940 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:04:41,940 DEBUG [RS:0;b60eac9bda43:34957 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b60eac9bda43,34957,1732122280382 2024-11-20T17:04:41,940 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b60eac9bda43,34957,1732122280382' 2024-11-20T17:04:41,940 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:04:41,941 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:04:41,941 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:04:41,941 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:04:41,941 DEBUG [RS:0;b60eac9bda43:34957 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b60eac9bda43,34957,1732122280382 2024-11-20T17:04:41,941 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b60eac9bda43,34957,1732122280382' 2024-11-20T17:04:41,941 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:04:41,942 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:04:41,942 DEBUG [RS:0;b60eac9bda43:34957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:04:41,942 INFO [RS:0;b60eac9bda43:34957 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:04:41,942 INFO [RS:0;b60eac9bda43:34957 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:04:41,971 WARN [b60eac9bda43:44751 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:04:42,046 INFO [RS:1;b60eac9bda43:36645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:04:42,046 INFO [RS:0;b60eac9bda43:34957 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:04:42,046 INFO [RS:2;b60eac9bda43:36999 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:04:42,051 INFO [RS:0;b60eac9bda43:34957 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C34957%2C1732122280382, suffix=, logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382, archiveDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs, maxLogs=32 2024-11-20T17:04:42,051 INFO [RS:2;b60eac9bda43:36999 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C36999%2C1732122280556, suffix=, logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36999,1732122280556, archiveDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs, maxLogs=32 2024-11-20T17:04:42,051 INFO [RS:1;b60eac9bda43:36645 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C36645%2C1732122280488, suffix=, logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36645,1732122280488, archiveDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs, maxLogs=32 2024-11-20T17:04:42,071 DEBUG [RS:1;b60eac9bda43:36645 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36645,1732122280488/b60eac9bda43%2C36645%2C1732122280488.1732122282057, exclude list is [], retry=0 2024-11-20T17:04:42,074 DEBUG [RS:2;b60eac9bda43:36999 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36999,1732122280556/b60eac9bda43%2C36999%2C1732122280556.1732122282056, exclude list is [], retry=0 2024-11-20T17:04:42,074 DEBUG [RS:0;b60eac9bda43:34957 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382/b60eac9bda43%2C34957%2C1732122280382.1732122282056, exclude list is [], retry=0 2024-11-20T17:04:42,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33493,DS-7004819e-1508-46b4-86e4-acb222d36d37,DISK] 2024-11-20T17:04:42,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-4e1882e0-2569-4b10-92a0-f5eec9f95d96,DISK] 2024-11-20T17:04:42,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41999,DS-db2b7c12-a09c-4d0d-aea8-9fea2ae5a604,DISK] 2024-11-20T17:04:42,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-4e1882e0-2569-4b10-92a0-f5eec9f95d96,DISK] 2024-11-20T17:04:42,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33493,DS-7004819e-1508-46b4-86e4-acb222d36d37,DISK] 2024-11-20T17:04:42,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41999,DS-db2b7c12-a09c-4d0d-aea8-9fea2ae5a604,DISK] 2024-11-20T17:04:42,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33493,DS-7004819e-1508-46b4-86e4-acb222d36d37,DISK] 2024-11-20T17:04:42,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-4e1882e0-2569-4b10-92a0-f5eec9f95d96,DISK] 2024-11-20T17:04:42,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41999,DS-db2b7c12-a09c-4d0d-aea8-9fea2ae5a604,DISK] 2024-11-20T17:04:42,082 INFO [RS:1;b60eac9bda43:36645 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36645,1732122280488/b60eac9bda43%2C36645%2C1732122280488.1732122282057 2024-11-20T17:04:42,083 DEBUG [RS:1;b60eac9bda43:36645 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33029:33029),(127.0.0.1/127.0.0.1:37891:37891),(127.0.0.1/127.0.0.1:45507:45507)] 2024-11-20T17:04:42,087 INFO [RS:0;b60eac9bda43:34957 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382/b60eac9bda43%2C34957%2C1732122280382.1732122282056 2024-11-20T17:04:42,087 INFO [RS:2;b60eac9bda43:36999 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36999,1732122280556/b60eac9bda43%2C36999%2C1732122280556.1732122282056 2024-11-20T17:04:42,087 DEBUG [RS:2;b60eac9bda43:36999 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37891:37891),(127.0.0.1/127.0.0.1:33029:33029),(127.0.0.1/127.0.0.1:45507:45507)] 2024-11-20T17:04:42,088 DEBUG [RS:0;b60eac9bda43:34957 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37891:37891),(127.0.0.1/127.0.0.1:45507:45507),(127.0.0.1/127.0.0.1:33029:33029)] 2024-11-20T17:04:42,224 DEBUG [b60eac9bda43:44751 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T17:04:42,233 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(204): Hosts are {b60eac9bda43=0} racks are {/default-rack=0} 2024-11-20T17:04:42,239 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:04:42,239 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:04:42,239 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T17:04:42,239 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:04:42,239 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:04:42,240 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T17:04:42,240 INFO [b60eac9bda43:44751 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:04:42,240 INFO [b60eac9bda43:44751 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:04:42,240 INFO [b60eac9bda43:44751 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T17:04:42,240 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:04:42,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:42,253 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,34957,1732122280382, state=OPENING 2024-11-20T17:04:42,307 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T17:04:42,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:42,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:42,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:42,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:42,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:42,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:42,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:42,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:42,317 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:42,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:42,499 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:04:42,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56417, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:04:42,514 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T17:04:42,514 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:04:42,515 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T17:04:42,518 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C34957%2C1732122280382.meta, suffix=.meta, logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382, archiveDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs, maxLogs=32 2024-11-20T17:04:42,534 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, exclude list is [], retry=0 2024-11-20T17:04:42,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33493,DS-7004819e-1508-46b4-86e4-acb222d36d37,DISK] 2024-11-20T17:04:42,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41999,DS-db2b7c12-a09c-4d0d-aea8-9fea2ae5a604,DISK] 2024-11-20T17:04:42,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-4e1882e0-2569-4b10-92a0-f5eec9f95d96,DISK] 2024-11-20T17:04:42,541 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta 2024-11-20T17:04:42,541 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33029:33029),(127.0.0.1/127.0.0.1:45507:45507),(127.0.0.1/127.0.0.1:37891:37891)] 2024-11-20T17:04:42,542 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:04:42,543 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T17:04:42,546 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T17:04:42,550 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T17:04:42,553 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T17:04:42,554 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:42,554 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T17:04:42,554 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T17:04:42,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T17:04:42,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:04:42,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:42,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:42,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T17:04:42,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:04:42,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:42,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:42,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T17:04:42,565 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:04:42,565 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:42,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:42,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T17:04:42,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:04:42,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:42,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:42,569 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:04:42,570 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740 2024-11-20T17:04:42,573 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740 2024-11-20T17:04:42,575 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:04:42,575 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:04:42,576 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:04:42,579 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:04:42,580 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60037165, jitterRate=-0.10537652671337128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:04:42,581 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T17:04:42,582 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732122282555Writing region info on filesystem at 1732122282555Initializing all the Stores at 1732122282557 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122282557Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122282557Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122282557Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122282557Cleaning up temporary data from old regions at 1732122282576 (+19 ms)Running coprocessor post-open hooks at 1732122282581 (+5 ms)Region opened successfully at 1732122282582 (+1 ms) 2024-11-20T17:04:42,588 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732122282488 2024-11-20T17:04:42,599 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T17:04:42,599 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T17:04:42,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:42,604 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,34957,1732122280382, state=OPEN 2024-11-20T17:04:42,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:42,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:42,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:42,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:42,640 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:42,640 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:42,640 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:42,640 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:42,640 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:42,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T17:04:42,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,34957,1732122280382 in 322 msec 2024-11-20T17:04:42,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T17:04:42,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 841 msec 2024-11-20T17:04:42,655 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:42,656 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T17:04:42,675 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:04:42,676 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1] 2024-11-20T17:04:42,694 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:04:42,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51515, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:04:42,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1620 sec 2024-11-20T17:04:42,718 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732122282717, completionTime=-1 2024-11-20T17:04:42,720 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T17:04:42,720 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:42,744 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-20T17:04:42,744 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122342744 2024-11-20T17:04:42,744 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122402744 2024-11-20T17:04:42,744 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-20T17:04:42,746 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:42,752 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,44751,1732122279653-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:42,752 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,44751,1732122279653-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:42,752 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,44751,1732122279653-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:42,754 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b60eac9bda43:44751, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:42,754 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:42,755 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:42,761 DEBUG [master/b60eac9bda43:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T17:04:42,780 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.120sec 2024-11-20T17:04:42,781 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T17:04:42,782 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T17:04:42,783 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T17:04:42,784 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T17:04:42,784 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T17:04:42,785 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,44751,1732122279653-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:04:42,786 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,44751,1732122279653-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T17:04:42,791 DEBUG [master/b60eac9bda43:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T17:04:42,792 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T17:04:42,793 INFO [master/b60eac9bda43:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b60eac9bda43,44751,1732122279653-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:04:42,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:04:42,804 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T17:04:42,804 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T17:04:42,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b60eac9bda43,44751,-1 for getting cluster id 2024-11-20T17:04:42,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:04:42,817 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3098b73-f92a-4174-8ea2-e2512d71554a' 2024-11-20T17:04:42,819 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:04:42,819 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3098b73-f92a-4174-8ea2-e2512d71554a" 2024-11-20T17:04:42,820 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a971440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:04:42,820 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b60eac9bda43,44751,-1] 2024-11-20T17:04:42,822 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:04:42,825 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:42,826 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:04:42,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:04:42,829 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:04:42,837 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1] 2024-11-20T17:04:42,838 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:04:42,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:04:42,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b60eac9bda43,44751,1732122279653 2024-11-20T17:04:42,880 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:04:42,885 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b60eac9bda43,44751,1732122279653 2024-11-20T17:04:42,887 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77bf91b5 2024-11-20T17:04:42,888 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:04:42,891 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:04:42,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:04:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:42,911 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:04:42,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-20T17:04:42,913 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:42,917 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:04:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:04:42,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741837_1013 (size=358) 2024-11-20T17:04:42,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741837_1013 (size=358) 2024-11-20T17:04:42,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741837_1013 (size=358) 2024-11-20T17:04:42,937 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a7a9afb4414c2ce282ecb8a226b732ef, NAME => 'test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:42,938 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0ae1354f6c79b7fe5dfb500ee1eb508d, NAME => 'test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:42,939 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9b1a96a3d9afa0a1d03650e42aa3e2fd, NAME => 'test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:42,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741839_1015 (size=45) 2024-11-20T17:04:42,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741839_1015 (size=45) 2024-11-20T17:04:42,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741839_1015 (size=45) 2024-11-20T17:04:42,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:42,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0ae1354f6c79b7fe5dfb500ee1eb508d, disabling compactions & flushes 2024-11-20T17:04:42,963 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:42,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:42,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. after waiting 0 ms 2024-11-20T17:04:42,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:42,963 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:42,963 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0ae1354f6c79b7fe5dfb500ee1eb508d: Waiting for close lock at 1732122282963Disabling compacts and flushes for region at 1732122282963Disabling writes for close at 1732122282963Writing region close event to WAL at 1732122282963Closed at 1732122282963 2024-11-20T17:04:42,964 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e0f5627de02ae3387d78a84cf60de116, NAME => 'test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:42,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741838_1014 (size=42) 2024-11-20T17:04:42,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741838_1014 (size=42) 2024-11-20T17:04:42,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741838_1014 (size=42) 2024-11-20T17:04:42,970 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:42,970 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a7a9afb4414c2ce282ecb8a226b732ef, disabling compactions & flushes 2024-11-20T17:04:42,970 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. 2024-11-20T17:04:42,970 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. 2024-11-20T17:04:42,971 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. after waiting 0 ms 2024-11-20T17:04:42,971 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. 2024-11-20T17:04:42,971 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. 2024-11-20T17:04:42,971 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a7a9afb4414c2ce282ecb8a226b732ef: Waiting for close lock at 1732122282970Disabling compacts and flushes for region at 1732122282970Disabling writes for close at 1732122282971 (+1 ms)Writing region close event to WAL at 1732122282971Closed at 1732122282971 2024-11-20T17:04:42,971 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5b4bd07e3663cc64a25dde8cd5845560, NAME => 'test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741840_1016 (size=45) 2024-11-20T17:04:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741840_1016 (size=45) 2024-11-20T17:04:42,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741840_1016 (size=45) 2024-11-20T17:04:42,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:42,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9b1a96a3d9afa0a1d03650e42aa3e2fd, disabling compactions & flushes 2024-11-20T17:04:42,985 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:42,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:42,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. after waiting 0 ms 2024-11-20T17:04:42,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:42,985 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:42,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9b1a96a3d9afa0a1d03650e42aa3e2fd: Waiting for close lock at 1732122282985Disabling compacts and flushes for region at 1732122282985Disabling writes for close at 1732122282985Writing region close event to WAL at 1732122282985Closed at 1732122282985 2024-11-20T17:04:42,986 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 12d417e0d4f0edac4d7bcadfbf305d96, NAME => 'test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:42,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741841_1017 (size=45) 2024-11-20T17:04:43,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741841_1017 (size=45) 2024-11-20T17:04:43,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741841_1017 (size=45) 2024-11-20T17:04:43,001 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,001 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e0f5627de02ae3387d78a84cf60de116, disabling compactions & flushes 2024-11-20T17:04:43,001 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:43,001 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:43,001 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. after waiting 0 ms 2024-11-20T17:04:43,001 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:43,002 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:43,002 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e0f5627de02ae3387d78a84cf60de116: Waiting for close lock at 1732122283001Disabling compacts and flushes for region at 1732122283001Disabling writes for close at 1732122283001Writing region close event to WAL at 1732122283001Closed at 1732122283001 2024-11-20T17:04:43,003 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ead0f5940987b7964950b8f83e68ce7d, NAME => 'test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741842_1018 (size=45) 2024-11-20T17:04:43,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741842_1018 (size=45) 2024-11-20T17:04:43,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741842_1018 (size=45) 2024-11-20T17:04:43,015 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,015 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5b4bd07e3663cc64a25dde8cd5845560, disabling compactions & flushes 2024-11-20T17:04:43,015 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560. 2024-11-20T17:04:43,015 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560. 2024-11-20T17:04:43,015 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560. after waiting 0 ms 2024-11-20T17:04:43,015 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560. 2024-11-20T17:04:43,015 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560. 2024-11-20T17:04:43,015 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5b4bd07e3663cc64a25dde8cd5845560: Waiting for close lock at 1732122283015Disabling compacts and flushes for region at 1732122283015Disabling writes for close at 1732122283015Writing region close event to WAL at 1732122283015Closed at 1732122283015 2024-11-20T17:04:43,016 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d48d4832d6d33017d258797576aabdc0, NAME => 'test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741843_1019 (size=45) 2024-11-20T17:04:43,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741843_1019 (size=45) 2024-11-20T17:04:43,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741843_1019 (size=45) 2024-11-20T17:04:43,027 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,027 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 12d417e0d4f0edac4d7bcadfbf305d96, disabling compactions & flushes 2024-11-20T17:04:43,027 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:43,028 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:43,028 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. after waiting 0 ms 2024-11-20T17:04:43,028 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:43,028 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:43,028 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 12d417e0d4f0edac4d7bcadfbf305d96: Waiting for close lock at 1732122283027Disabling compacts and flushes for region at 1732122283027Disabling writes for close at 1732122283028 (+1 ms)Writing region close event to WAL at 1732122283028Closed at 1732122283028 2024-11-20T17:04:43,028 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 736a1f385c939ce1414e8e279f4e5834, NAME => 'test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:04:43,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741844_1020 (size=45) 2024-11-20T17:04:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741844_1020 (size=45) 2024-11-20T17:04:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741844_1020 (size=45) 2024-11-20T17:04:43,045 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,045 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ead0f5940987b7964950b8f83e68ce7d, disabling compactions & flushes 2024-11-20T17:04:43,045 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:43,045 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:43,045 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. after waiting 0 ms 2024-11-20T17:04:43,045 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:43,045 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:43,046 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ead0f5940987b7964950b8f83e68ce7d: Waiting for close lock at 1732122283045Disabling compacts and flushes for region at 1732122283045Disabling writes for close at 1732122283045Writing region close event to WAL at 1732122283045Closed at 1732122283045 2024-11-20T17:04:43,046 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => be724bb9504bda880d6a757168537eb1, NAME => 'test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741845_1021 (size=45) 2024-11-20T17:04:43,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741846_1022 (size=45) 2024-11-20T17:04:43,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741845_1021 (size=45) 2024-11-20T17:04:43,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741846_1022 (size=45) 2024-11-20T17:04:43,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741845_1021 (size=45) 2024-11-20T17:04:43,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741846_1022 (size=45) 2024-11-20T17:04:43,068 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,068 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d48d4832d6d33017d258797576aabdc0, disabling compactions & flushes 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 736a1f385c939ce1414e8e279f4e5834, disabling compactions & flushes 2024-11-20T17:04:43,069 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:43,069 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. after waiting 0 ms 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. after waiting 0 ms 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:43,069 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:43,069 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d48d4832d6d33017d258797576aabdc0: Waiting for close lock at 1732122283068Disabling compacts and flushes for region at 1732122283068Disabling writes for close at 1732122283069 (+1 ms)Writing region close event to WAL at 1732122283069Closed at 1732122283069 2024-11-20T17:04:43,069 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 736a1f385c939ce1414e8e279f4e5834: Waiting for close lock at 1732122283068Disabling compacts and flushes for region at 1732122283068Disabling writes for close at 1732122283069 (+1 ms)Writing region close event to WAL at 1732122283069Closed at 1732122283069 2024-11-20T17:04:43,070 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f5cc1a8141b65dd3101cc2eb478bf7fc, NAME => 'test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,070 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0e0f80ef67003cc41131beab8b406f39, NAME => 'test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741847_1023 (size=45) 2024-11-20T17:04:43,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741847_1023 (size=45) 2024-11-20T17:04:43,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741847_1023 (size=45) 2024-11-20T17:04:43,087 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,088 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing be724bb9504bda880d6a757168537eb1, disabling compactions & flushes 2024-11-20T17:04:43,088 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:43,088 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:43,088 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. after waiting 0 ms 2024-11-20T17:04:43,088 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:43,088 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:43,088 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for be724bb9504bda880d6a757168537eb1: Waiting for close lock at 1732122283088Disabling compacts and flushes for region at 1732122283088Disabling writes for close at 1732122283088Writing region close event to WAL at 1732122283088Closed at 1732122283088 2024-11-20T17:04:43,089 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d9dc92a1bd5b9a54b65f86671af0015f, NAME => 'test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741849_1025 (size=45) 2024-11-20T17:04:43,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741849_1025 (size=45) 2024-11-20T17:04:43,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741849_1025 (size=45) 2024-11-20T17:04:43,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 0e0f80ef67003cc41131beab8b406f39, disabling compactions & flushes 2024-11-20T17:04:43,104 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39. 2024-11-20T17:04:43,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39. 2024-11-20T17:04:43,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741848_1024 (size=45) 2024-11-20T17:04:43,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39. after waiting 0 ms 2024-11-20T17:04:43,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39. 2024-11-20T17:04:43,104 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39. 2024-11-20T17:04:43,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0e0f80ef67003cc41131beab8b406f39: Waiting for close lock at 1732122283104Disabling compacts and flushes for region at 1732122283104Disabling writes for close at 1732122283104Writing region close event to WAL at 1732122283104Closed at 1732122283104 2024-11-20T17:04:43,105 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 052ad436a3fbddb02bb8de2342c3b9a7, NAME => 'test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741848_1024 (size=45) 2024-11-20T17:04:43,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741848_1024 (size=45) 2024-11-20T17:04:43,108 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f5cc1a8141b65dd3101cc2eb478bf7fc, disabling compactions & flushes 2024-11-20T17:04:43,109 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:43,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:43,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. after waiting 0 ms 2024-11-20T17:04:43,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:43,109 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:43,109 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f5cc1a8141b65dd3101cc2eb478bf7fc: Waiting for close lock at 1732122283109Disabling compacts and flushes for region at 1732122283109Disabling writes for close at 1732122283109Writing region close event to WAL at 1732122283109Closed at 1732122283109 2024-11-20T17:04:43,110 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 13fb0e6117ef98d16b8be8bc4a741d1b, NAME => 'test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741850_1026 (size=45) 2024-11-20T17:04:43,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741850_1026 (size=45) 2024-11-20T17:04:43,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741850_1026 (size=45) 2024-11-20T17:04:43,126 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,126 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d9dc92a1bd5b9a54b65f86671af0015f, disabling compactions & flushes 2024-11-20T17:04:43,126 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:43,126 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:43,126 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. after waiting 0 ms 2024-11-20T17:04:43,126 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:43,126 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:43,126 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d9dc92a1bd5b9a54b65f86671af0015f: Waiting for close lock at 1732122283126Disabling compacts and flushes for region at 1732122283126Disabling writes for close at 1732122283126Writing region close event to WAL at 1732122283126Closed at 1732122283126 2024-11-20T17:04:43,127 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ad24e48b8f4e256db964da467a470ef8, NAME => 'test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741851_1027 (size=45) 2024-11-20T17:04:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741851_1027 (size=45) 2024-11-20T17:04:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741851_1027 (size=45) 2024-11-20T17:04:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741852_1028 (size=45) 2024-11-20T17:04:43,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741852_1028 (size=45) 2024-11-20T17:04:43,132 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,132 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 052ad436a3fbddb02bb8de2342c3b9a7, disabling compactions & flushes 2024-11-20T17:04:43,132 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:43,132 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:43,133 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. after waiting 0 ms 2024-11-20T17:04:43,133 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:43,133 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:43,133 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 052ad436a3fbddb02bb8de2342c3b9a7: Waiting for close lock at 1732122283132Disabling compacts and flushes for region at 1732122283132Disabling writes for close at 1732122283133 (+1 ms)Writing region close event to WAL at 1732122283133Closed at 1732122283133 2024-11-20T17:04:43,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741852_1028 (size=45) 2024-11-20T17:04:43,133 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b830d024dd19ff173c226582a1d0e05b, NAME => 'test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 13fb0e6117ef98d16b8be8bc4a741d1b, disabling compactions & flushes 2024-11-20T17:04:43,134 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. after waiting 0 ms 2024-11-20T17:04:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:43,134 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:43,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 13fb0e6117ef98d16b8be8bc4a741d1b: Waiting for close lock at 1732122283134Disabling compacts and flushes for region at 1732122283134Disabling writes for close at 1732122283134Writing region close event to WAL at 1732122283134Closed at 1732122283134 2024-11-20T17:04:43,135 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9de6e6c862796cabf8368e339eb39e67, NAME => 'test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741853_1029 (size=45) 2024-11-20T17:04:43,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741853_1029 (size=45) 2024-11-20T17:04:43,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741853_1029 (size=45) 2024-11-20T17:04:43,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ad24e48b8f4e256db964da467a470ef8, disabling compactions & flushes 2024-11-20T17:04:43,151 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:43,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:43,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. after waiting 0 ms 2024-11-20T17:04:43,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:43,151 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:43,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ad24e48b8f4e256db964da467a470ef8: Waiting for close lock at 1732122283151Disabling compacts and flushes for region at 1732122283151Disabling writes for close at 1732122283151Writing region close event to WAL at 1732122283151Closed at 1732122283151 2024-11-20T17:04:43,152 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c22acb23a82c5ea894235ab5c0c16804, NAME => 'test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741854_1030 (size=45) 2024-11-20T17:04:43,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741854_1030 (size=45) 2024-11-20T17:04:43,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741854_1030 (size=45) 2024-11-20T17:04:43,160 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,160 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b830d024dd19ff173c226582a1d0e05b, disabling compactions & flushes 2024-11-20T17:04:43,160 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b. 2024-11-20T17:04:43,160 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b. 2024-11-20T17:04:43,160 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b. after waiting 0 ms 2024-11-20T17:04:43,160 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b. 2024-11-20T17:04:43,160 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b. 2024-11-20T17:04:43,160 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b830d024dd19ff173c226582a1d0e05b: Waiting for close lock at 1732122283160Disabling compacts and flushes for region at 1732122283160Disabling writes for close at 1732122283160Writing region close event to WAL at 1732122283160Closed at 1732122283160 2024-11-20T17:04:43,161 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7b8c248055134523e6b12d02a033d557, NAME => 'test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741855_1031 (size=45) 2024-11-20T17:04:43,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741855_1031 (size=45) 2024-11-20T17:04:43,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741855_1031 (size=45) 2024-11-20T17:04:43,171 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,171 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9de6e6c862796cabf8368e339eb39e67, disabling compactions & flushes 2024-11-20T17:04:43,171 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:43,171 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:43,171 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. after waiting 0 ms 2024-11-20T17:04:43,171 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:43,171 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:43,171 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9de6e6c862796cabf8368e339eb39e67: Waiting for close lock at 1732122283171Disabling compacts and flushes for region at 1732122283171Disabling writes for close at 1732122283171Writing region close event to WAL at 1732122283171Closed at 1732122283171 2024-11-20T17:04:43,172 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7d0e6a5001fbe78b88c20912afb21151, NAME => 'test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741856_1032 (size=45) 2024-11-20T17:04:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741856_1032 (size=45) 2024-11-20T17:04:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741856_1032 (size=45) 2024-11-20T17:04:43,191 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,192 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c22acb23a82c5ea894235ab5c0c16804, disabling compactions & flushes 2024-11-20T17:04:43,192 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:43,192 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:43,192 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. after waiting 0 ms 2024-11-20T17:04:43,192 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:43,192 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:43,192 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c22acb23a82c5ea894235ab5c0c16804: Waiting for close lock at 1732122283192Disabling compacts and flushes for region at 1732122283192Disabling writes for close at 1732122283192Writing region close event to WAL at 1732122283192Closed at 1732122283192 2024-11-20T17:04:43,193 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c7bdef0be62b91bb7a8d2486eee55f68, NAME => 'test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741857_1033 (size=45) 2024-11-20T17:04:43,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741857_1033 (size=45) 2024-11-20T17:04:43,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741857_1033 (size=45) 2024-11-20T17:04:43,194 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,195 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7b8c248055134523e6b12d02a033d557, disabling compactions & flushes 2024-11-20T17:04:43,195 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:43,195 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:43,195 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. after waiting 0 ms 2024-11-20T17:04:43,195 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:43,195 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:43,195 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7b8c248055134523e6b12d02a033d557: Waiting for close lock at 1732122283194Disabling compacts and flushes for region at 1732122283194Disabling writes for close at 1732122283195 (+1 ms)Writing region close event to WAL at 1732122283195Closed at 1732122283195 2024-11-20T17:04:43,195 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d01fc4378925287cb15b42165d6247f2, NAME => 'test,www,1732122282892.d01fc4378925287cb15b42165d6247f2.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741858_1034 (size=45) 2024-11-20T17:04:43,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741858_1034 (size=45) 2024-11-20T17:04:43,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741858_1034 (size=45) 2024-11-20T17:04:43,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7d0e6a5001fbe78b88c20912afb21151, disabling compactions & flushes 2024-11-20T17:04:43,206 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:43,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:43,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. after waiting 0 ms 2024-11-20T17:04:43,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:43,206 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:43,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7d0e6a5001fbe78b88c20912afb21151: Waiting for close lock at 1732122283206Disabling compacts and flushes for region at 1732122283206Disabling writes for close at 1732122283206Writing region close event to WAL at 1732122283206Closed at 1732122283206 2024-11-20T17:04:43,207 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 17a1952b1af83907647c5d82cf786cc3, NAME => 'test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741859_1035 (size=45) 2024-11-20T17:04:43,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741859_1035 (size=45) 2024-11-20T17:04:43,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741860_1036 (size=45) 2024-11-20T17:04:43,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741860_1036 (size=45) 2024-11-20T17:04:43,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741860_1036 (size=45) 2024-11-20T17:04:43,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741859_1035 (size=45) 2024-11-20T17:04:43,211 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c7bdef0be62b91bb7a8d2486eee55f68, disabling compactions & flushes 2024-11-20T17:04:43,212 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732122282892.d01fc4378925287cb15b42165d6247f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. after waiting 0 ms 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d01fc4378925287cb15b42165d6247f2, disabling compactions & flushes 2024-11-20T17:04:43,212 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:43,212 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c7bdef0be62b91bb7a8d2486eee55f68: Waiting for close lock at 1732122283212Disabling compacts and flushes for region at 1732122283212Disabling writes for close at 1732122283212Writing region close event to WAL at 1732122283212Closed at 1732122283212 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. after waiting 0 ms 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. 2024-11-20T17:04:43,212 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. 2024-11-20T17:04:43,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d01fc4378925287cb15b42165d6247f2: Waiting for close lock at 1732122283212Disabling compacts and flushes for region at 1732122283212Disabling writes for close at 1732122283212Writing region close event to WAL at 1732122283212Closed at 1732122283212 2024-11-20T17:04:43,212 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fd9711911c68abdd7f4a5f7d15d952e9, NAME => 'test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,213 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c41b1abb64df3eda0e1232a964811d27, NAME => 'test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e 2024-11-20T17:04:43,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741861_1037 (size=45) 2024-11-20T17:04:43,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741861_1037 (size=45) 2024-11-20T17:04:43,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741861_1037 (size=45) 2024-11-20T17:04:43,224 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,224 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 17a1952b1af83907647c5d82cf786cc3, disabling compactions & flushes 2024-11-20T17:04:43,224 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:43,224 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:43,224 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. after waiting 0 ms 2024-11-20T17:04:43,224 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:43,224 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:43,224 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 17a1952b1af83907647c5d82cf786cc3: Waiting for close lock at 1732122283224Disabling compacts and flushes for region at 1732122283224Disabling writes for close at 1732122283224Writing region close event to WAL at 1732122283224Closed at 1732122283224 2024-11-20T17:04:43,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741863_1039 (size=42) 2024-11-20T17:04:43,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741863_1039 (size=42) 2024-11-20T17:04:43,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741862_1038 (size=45) 2024-11-20T17:04:43,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741863_1039 (size=42) 2024-11-20T17:04:43,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741862_1038 (size=45) 2024-11-20T17:04:43,226 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,226 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c41b1abb64df3eda0e1232a964811d27, disabling compactions & flushes 2024-11-20T17:04:43,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741862_1038 (size=45) 2024-11-20T17:04:43,226 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:43,226 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:43,226 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. after waiting 0 ms 2024-11-20T17:04:43,226 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:43,227 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:43,227 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c41b1abb64df3eda0e1232a964811d27: Waiting for close lock at 1732122283226Disabling compacts and flushes for region at 1732122283226Disabling writes for close at 1732122283226Writing region close event to WAL at 1732122283227 (+1 ms)Closed at 1732122283227 2024-11-20T17:04:43,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fd9711911c68abdd7f4a5f7d15d952e9, disabling compactions & flushes 2024-11-20T17:04:43,228 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:43,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:43,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. after waiting 0 ms 2024-11-20T17:04:43,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:43,228 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:43,228 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fd9711911c68abdd7f4a5f7d15d952e9: Waiting for close lock at 1732122283228Disabling compacts and flushes for region at 1732122283228Disabling writes for close at 1732122283228Writing region close event to WAL at 1732122283228Closed at 1732122283228 2024-11-20T17:04:43,231 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:04:43,236 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,237 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,237 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,237 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,237 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,237 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,239 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,239 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,239 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,239 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,240 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,240 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,240 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,240 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,240 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,241 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,241 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,241 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,241 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732122282892.d01fc4378925287cb15b42165d6247f2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,242 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,242 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,242 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732122283231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732122283231"}]},"ts":"1732122283231"} 2024-11-20T17:04:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:04:43,292 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-20T17:04:43,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:04:43,298 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732122283295"}]},"ts":"1732122283295"} 2024-11-20T17:04:43,303 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-20T17:04:43,304 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {b60eac9bda43=0} racks are {/default-rack=0} 2024-11-20T17:04:43,322 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:04:43,322 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:04:43,322 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T17:04:43,322 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:04:43,322 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:04:43,322 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T17:04:43,322 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:04:43,323 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:04:43,323 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T17:04:43,323 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:04:43,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0ae1354f6c79b7fe5dfb500ee1eb508d, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b1a96a3d9afa0a1d03650e42aa3e2fd, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0f5627de02ae3387d78a84cf60de116, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12d417e0d4f0edac4d7bcadfbf305d96, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ead0f5940987b7964950b8f83e68ce7d, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=736a1f385c939ce1414e8e279f4e5834, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=be724bb9504bda880d6a757168537eb1, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5cc1a8141b65dd3101cc2eb478bf7fc, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d9dc92a1bd5b9a54b65f86671af0015f, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=13fb0e6117ef98d16b8be8bc4a741d1b, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad24e48b8f4e256db964da467a470ef8, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9de6e6c862796cabf8368e339eb39e67, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c22acb23a82c5ea894235ab5c0c16804, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d0e6a5001fbe78b88c20912afb21151, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c7bdef0be62b91bb7a8d2486eee55f68, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17a1952b1af83907647c5d82cf786cc3, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c41b1abb64df3eda0e1232a964811d27, ASSIGN}] 2024-11-20T17:04:43,334 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c41b1abb64df3eda0e1232a964811d27, ASSIGN 2024-11-20T17:04:43,336 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c41b1abb64df3eda0e1232a964811d27, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,337 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, ASSIGN 2024-11-20T17:04:43,338 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,340 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17a1952b1af83907647c5d82cf786cc3, ASSIGN 2024-11-20T17:04:43,342 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=17a1952b1af83907647c5d82cf786cc3, ASSIGN; state=OFFLINE, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:43,343 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:43,345 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,346 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c7bdef0be62b91bb7a8d2486eee55f68, ASSIGN 2024-11-20T17:04:43,348 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c7bdef0be62b91bb7a8d2486eee55f68, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,350 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d0e6a5001fbe78b88c20912afb21151, ASSIGN 2024-11-20T17:04:43,351 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7d0e6a5001fbe78b88c20912afb21151, ASSIGN; state=OFFLINE, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:43,353 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:43,355 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,356 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c22acb23a82c5ea894235ab5c0c16804, ASSIGN 2024-11-20T17:04:43,358 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c22acb23a82c5ea894235ab5c0c16804, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,360 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9de6e6c862796cabf8368e339eb39e67, ASSIGN 2024-11-20T17:04:43,362 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9de6e6c862796cabf8368e339eb39e67, ASSIGN; state=OFFLINE, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:43,364 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:43,366 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,368 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad24e48b8f4e256db964da467a470ef8, ASSIGN 2024-11-20T17:04:43,369 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ad24e48b8f4e256db964da467a470ef8, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,371 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=13fb0e6117ef98d16b8be8bc4a741d1b, ASSIGN 2024-11-20T17:04:43,373 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=13fb0e6117ef98d16b8be8bc4a741d1b, ASSIGN; state=OFFLINE, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:43,375 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:43,377 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,379 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d9dc92a1bd5b9a54b65f86671af0015f, ASSIGN 2024-11-20T17:04:43,380 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d9dc92a1bd5b9a54b65f86671af0015f, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,382 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:43,383 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,384 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5cc1a8141b65dd3101cc2eb478bf7fc, ASSIGN 2024-11-20T17:04:43,386 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5cc1a8141b65dd3101cc2eb478bf7fc, ASSIGN; state=OFFLINE, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:43,387 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=be724bb9504bda880d6a757168537eb1, ASSIGN 2024-11-20T17:04:43,388 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=be724bb9504bda880d6a757168537eb1, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,390 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=736a1f385c939ce1414e8e279f4e5834, ASSIGN 2024-11-20T17:04:43,391 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=736a1f385c939ce1414e8e279f4e5834, ASSIGN; state=OFFLINE, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:43,392 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:43,393 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,394 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ead0f5940987b7964950b8f83e68ce7d, ASSIGN 2024-11-20T17:04:43,395 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ead0f5940987b7964950b8f83e68ce7d, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,396 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12d417e0d4f0edac4d7bcadfbf305d96, ASSIGN 2024-11-20T17:04:43,397 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12d417e0d4f0edac4d7bcadfbf305d96, ASSIGN; state=OFFLINE, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:43,398 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:43,399 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,400 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e0f5627de02ae3387d78a84cf60de116, ASSIGN 2024-11-20T17:04:43,402 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e0f5627de02ae3387d78a84cf60de116, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,403 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b1a96a3d9afa0a1d03650e42aa3e2fd, ASSIGN 2024-11-20T17:04:43,404 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9b1a96a3d9afa0a1d03650e42aa3e2fd, ASSIGN; state=OFFLINE, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:43,405 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0ae1354f6c79b7fe5dfb500ee1eb508d, ASSIGN 2024-11-20T17:04:43,406 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0ae1354f6c79b7fe5dfb500ee1eb508d, ASSIGN; state=OFFLINE, location=b60eac9bda43,36645,1732122280488; forceNewPlan=false, retain=false 2024-11-20T17:04:43,408 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:43,410 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN; state=OFFLINE, location=b60eac9bda43,34957,1732122280382; forceNewPlan=false, retain=false 2024-11-20T17:04:43,491 INFO [b60eac9bda43:44751 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-20T17:04:43,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=17a1952b1af83907647c5d82cf786cc3, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=7d0e6a5001fbe78b88c20912afb21151, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9de6e6c862796cabf8368e339eb39e67, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=736a1f385c939ce1414e8e279f4e5834, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=17a1952b1af83907647c5d82cf786cc3, ASSIGN because future has completed 2024-11-20T17:04:43,500 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,500 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c41b1abb64df3eda0e1232a964811d27, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c22acb23a82c5ea894235ab5c0c16804, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ad24e48b8f4e256db964da467a470ef8, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17a1952b1af83907647c5d82cf786cc3, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:43,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7d0e6a5001fbe78b88c20912afb21151, ASSIGN because future has completed 2024-11-20T17:04:43,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9de6e6c862796cabf8368e339eb39e67, ASSIGN because future has completed 2024-11-20T17:04:43,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=13fb0e6117ef98d16b8be8bc4a741d1b, ASSIGN because future has completed 2024-11-20T17:04:43,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13fb0e6117ef98d16b8be8bc4a741d1b, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:43,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5cc1a8141b65dd3101cc2eb478bf7fc, ASSIGN because future has completed 2024-11-20T17:04:43,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=736a1f385c939ce1414e8e279f4e5834, ASSIGN because future has completed 2024-11-20T17:04:43,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12d417e0d4f0edac4d7bcadfbf305d96, ASSIGN because future has completed 2024-11-20T17:04:43,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12d417e0d4f0edac4d7bcadfbf305d96, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:43,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9b1a96a3d9afa0a1d03650e42aa3e2fd, ASSIGN because future has completed 2024-11-20T17:04:43,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c41b1abb64df3eda0e1232a964811d27, ASSIGN because future has completed 2024-11-20T17:04:43,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c41b1abb64df3eda0e1232a964811d27, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c7bdef0be62b91bb7a8d2486eee55f68, ASSIGN because future has completed 2024-11-20T17:04:43,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c22acb23a82c5ea894235ab5c0c16804, ASSIGN because future has completed 2024-11-20T17:04:43,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ad24e48b8f4e256db964da467a470ef8, ASSIGN because future has completed 2024-11-20T17:04:43,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad24e48b8f4e256db964da467a470ef8, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d9dc92a1bd5b9a54b65f86671af0015f, ASSIGN because future has completed 2024-11-20T17:04:43,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9dc92a1bd5b9a54b65f86671af0015f, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c22acb23a82c5ea894235ab5c0c16804, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7bdef0be62b91bb7a8d2486eee55f68, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:04:43,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b1a96a3d9afa0a1d03650e42aa3e2fd, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:43,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 736a1f385c939ce1414e8e279f4e5834, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:43,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5cc1a8141b65dd3101cc2eb478bf7fc, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:43,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9de6e6c862796cabf8368e339eb39e67, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:43,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d0e6a5001fbe78b88c20912afb21151, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:43,583 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=be724bb9504bda880d6a757168537eb1, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,584 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ead0f5940987b7964950b8f83e68ce7d, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e0f5627de02ae3387d78a84cf60de116, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,587 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=7b8c248055134523e6b12d02a033d557, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=be724bb9504bda880d6a757168537eb1, ASSIGN because future has completed 2024-11-20T17:04:43,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure be724bb9504bda880d6a757168537eb1, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ead0f5940987b7964950b8f83e68ce7d, ASSIGN because future has completed 2024-11-20T17:04:43,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e0f5627de02ae3387d78a84cf60de116, ASSIGN because future has completed 2024-11-20T17:04:43,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0ae1354f6c79b7fe5dfb500ee1eb508d, ASSIGN because future has completed 2024-11-20T17:04:43,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, ASSIGN because future has completed 2024-11-20T17:04:43,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN because future has completed 2024-11-20T17:04:43,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN because future has completed 2024-11-20T17:04:43,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ae1354f6c79b7fe5dfb500ee1eb508d, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0f5627de02ae3387d78a84cf60de116, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ead0f5940987b7964950b8f83e68ce7d, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:43,626 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,626 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,629 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN because future has completed 2024-11-20T17:04:43,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN because future has completed 2024-11-20T17:04:43,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN because future has completed 2024-11-20T17:04:43,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN because future has completed 2024-11-20T17:04:43,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN because future has completed 2024-11-20T17:04:43,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure d48d4832d6d33017d258797576aabdc0, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,648 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN because future has completed 2024-11-20T17:04:43,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:43,658 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:04:43,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56405, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:04:43,665 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:43,666 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => f5cc1a8141b65dd3101cc2eb478bf7fc, NAME => 'test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-20T17:04:43,666 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,666 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,666 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,666 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,668 INFO [StoreOpener-f5cc1a8141b65dd3101cc2eb478bf7fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,670 INFO [StoreOpener-f5cc1a8141b65dd3101cc2eb478bf7fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5cc1a8141b65dd3101cc2eb478bf7fc columnFamilyName family 2024-11-20T17:04:43,670 DEBUG [StoreOpener-f5cc1a8141b65dd3101cc2eb478bf7fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,671 INFO [StoreOpener-f5cc1a8141b65dd3101cc2eb478bf7fc-1 {}] regionserver.HStore(327): Store=f5cc1a8141b65dd3101cc2eb478bf7fc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,671 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,672 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,673 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,674 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,674 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,676 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,679 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/f5cc1a8141b65dd3101cc2eb478bf7fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,680 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened f5cc1a8141b65dd3101cc2eb478bf7fc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67668753, jitterRate=0.00834299623966217}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,680 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:43,682 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:04:43,682 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for f5cc1a8141b65dd3101cc2eb478bf7fc: Running coprocessor pre-open hook at 1732122283667Writing region info on filesystem at 1732122283667Initializing all the Stores at 1732122283668 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283668Cleaning up temporary data from old regions at 1732122283674 (+6 ms)Running coprocessor post-open hooks at 1732122283680 (+6 ms)Region opened successfully at 1732122283682 (+2 ms) 2024-11-20T17:04:43,684 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc., pid=41, masterSystemTime=1732122283658 2024-11-20T17:04:43,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52887, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:04:43,688 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:43,688 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:43,688 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:43,689 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 13fb0e6117ef98d16b8be8bc4a741d1b, NAME => 'test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-20T17:04:43,689 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,689 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,690 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,690 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,691 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:43,691 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => c22acb23a82c5ea894235ab5c0c16804, NAME => 'test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-20T17:04:43,692 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,692 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,692 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,692 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,692 INFO [StoreOpener-13fb0e6117ef98d16b8be8bc4a741d1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5cc1a8141b65dd3101cc2eb478bf7fc, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:43,694 INFO [StoreOpener-c22acb23a82c5ea894235ab5c0c16804-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,695 INFO [StoreOpener-13fb0e6117ef98d16b8be8bc4a741d1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13fb0e6117ef98d16b8be8bc4a741d1b columnFamilyName family 2024-11-20T17:04:43,695 DEBUG [StoreOpener-13fb0e6117ef98d16b8be8bc4a741d1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,696 INFO [StoreOpener-13fb0e6117ef98d16b8be8bc4a741d1b-1 {}] regionserver.HStore(327): Store=13fb0e6117ef98d16b8be8bc4a741d1b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,696 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,697 INFO [StoreOpener-c22acb23a82c5ea894235ab5c0c16804-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c22acb23a82c5ea894235ab5c0c16804 columnFamilyName family 2024-11-20T17:04:43,697 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,697 DEBUG [StoreOpener-c22acb23a82c5ea894235ab5c0c16804-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,698 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,698 INFO [StoreOpener-c22acb23a82c5ea894235ab5c0c16804-1 {}] regionserver.HStore(327): Store=c22acb23a82c5ea894235ab5c0c16804/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,698 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,698 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,698 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,699 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,700 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=15 2024-11-20T17:04:43,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure f5cc1a8141b65dd3101cc2eb478bf7fc, server=b60eac9bda43,36999,1732122280556 in 127 msec 2024-11-20T17:04:43,700 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,700 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,701 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f5cc1a8141b65dd3101cc2eb478bf7fc, ASSIGN in 375 msec 2024-11-20T17:04:43,704 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,704 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/13fb0e6117ef98d16b8be8bc4a741d1b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,705 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 13fb0e6117ef98d16b8be8bc4a741d1b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67754175, jitterRate=0.009615883231163025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,705 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:43,705 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 13fb0e6117ef98d16b8be8bc4a741d1b: Running coprocessor pre-open hook at 1732122283690Writing region info on filesystem at 1732122283690Initializing all the Stores at 1732122283692 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283692Cleaning up temporary data from old regions at 1732122283698 (+6 ms)Running coprocessor post-open hooks at 1732122283705 (+7 ms)Region opened successfully at 1732122283705 2024-11-20T17:04:43,707 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b., pid=32, masterSystemTime=1732122283658 2024-11-20T17:04:43,707 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c22acb23a82c5ea894235ab5c0c16804/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,708 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened c22acb23a82c5ea894235ab5c0c16804; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59521135, jitterRate=-0.11306597292423248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,708 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:43,708 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for c22acb23a82c5ea894235ab5c0c16804: Running coprocessor pre-open hook at 1732122283692Writing region info on filesystem at 1732122283692Initializing all the Stores at 1732122283694 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283694Cleaning up temporary data from old regions at 1732122283700 (+6 ms)Running coprocessor post-open hooks at 1732122283708 (+8 ms)Region opened successfully at 1732122283708 2024-11-20T17:04:43,709 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804., pid=37, masterSystemTime=1732122283681 2024-11-20T17:04:43,710 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:43,710 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:43,711 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:43,711 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b1a96a3d9afa0a1d03650e42aa3e2fd, NAME => 'test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-20T17:04:43,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,712 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,712 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,712 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,712 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,713 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:43,714 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:43,714 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:43,714 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => ead0f5940987b7964950b8f83e68ce7d, NAME => 'test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-20T17:04:43,715 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,715 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,715 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,715 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,715 INFO [StoreOpener-9b1a96a3d9afa0a1d03650e42aa3e2fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13fb0e6117ef98d16b8be8bc4a741d1b, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:43,716 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,718 INFO [StoreOpener-ead0f5940987b7964950b8f83e68ce7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c22acb23a82c5ea894235ab5c0c16804, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:43,720 INFO [StoreOpener-9b1a96a3d9afa0a1d03650e42aa3e2fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b1a96a3d9afa0a1d03650e42aa3e2fd columnFamilyName family 2024-11-20T17:04:43,721 DEBUG [StoreOpener-9b1a96a3d9afa0a1d03650e42aa3e2fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,722 INFO [StoreOpener-9b1a96a3d9afa0a1d03650e42aa3e2fd-1 {}] regionserver.HStore(327): Store=9b1a96a3d9afa0a1d03650e42aa3e2fd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,722 INFO [StoreOpener-ead0f5940987b7964950b8f83e68ce7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ead0f5940987b7964950b8f83e68ce7d columnFamilyName family 2024-11-20T17:04:43,722 DEBUG [StoreOpener-ead0f5940987b7964950b8f83e68ce7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,722 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,723 INFO [StoreOpener-ead0f5940987b7964950b8f83e68ce7d-1 {}] regionserver.HStore(327): Store=ead0f5940987b7964950b8f83e68ce7d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,723 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,723 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,724 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=19 2024-11-20T17:04:43,724 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 13fb0e6117ef98d16b8be8bc4a741d1b, server=b60eac9bda43,36999,1732122280556 in 208 msec 2024-11-20T17:04:43,724 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,724 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,725 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,726 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,726 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,727 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=13fb0e6117ef98d16b8be8bc4a741d1b, ASSIGN in 399 msec 2024-11-20T17:04:43,728 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,731 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/9b1a96a3d9afa0a1d03650e42aa3e2fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,732 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 9b1a96a3d9afa0a1d03650e42aa3e2fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67001221, jitterRate=-0.0016040056943893433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,732 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:43,732 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 9b1a96a3d9afa0a1d03650e42aa3e2fd: Running coprocessor pre-open hook at 1732122283712Writing region info on filesystem at 1732122283712Initializing all the Stores at 1732122283714 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283714Cleaning up temporary data from old regions at 1732122283725 (+11 ms)Running coprocessor post-open hooks at 1732122283732 (+7 ms)Region opened successfully at 1732122283732 2024-11-20T17:04:43,733 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd., pid=39, masterSystemTime=1732122283658 2024-11-20T17:04:43,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=23 2024-11-20T17:04:43,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure c22acb23a82c5ea894235ab5c0c16804, server=b60eac9bda43,36645,1732122280488 in 185 msec 2024-11-20T17:04:43,735 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/ead0f5940987b7964950b8f83e68ce7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,736 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:43,736 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:43,737 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:43,737 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened ead0f5940987b7964950b8f83e68ce7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68699558, jitterRate=0.023703187704086304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,737 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:43,737 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 736a1f385c939ce1414e8e279f4e5834, NAME => 'test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-20T17:04:43,737 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for ead0f5940987b7964950b8f83e68ce7d: Running coprocessor pre-open hook at 1732122283715Writing region info on filesystem at 1732122283715Initializing all the Stores at 1732122283717 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283717Cleaning up temporary data from old regions at 1732122283726 (+9 ms)Running coprocessor post-open hooks at 1732122283737 (+11 ms)Region opened successfully at 1732122283737 2024-11-20T17:04:43,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c22acb23a82c5ea894235ab5c0c16804, ASSIGN in 409 msec 2024-11-20T17:04:43,737 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,737 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,737 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,738 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,738 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d., pid=50, masterSystemTime=1732122283681 2024-11-20T17:04:43,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,740 INFO [StoreOpener-736a1f385c939ce1414e8e279f4e5834-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,741 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:43,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b1a96a3d9afa0a1d03650e42aa3e2fd, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:43,741 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:43,741 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:43,742 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ae1354f6c79b7fe5dfb500ee1eb508d, NAME => 'test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-20T17:04:43,742 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,742 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,742 INFO [StoreOpener-736a1f385c939ce1414e8e279f4e5834-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 736a1f385c939ce1414e8e279f4e5834 columnFamilyName family 2024-11-20T17:04:43,742 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,742 DEBUG [StoreOpener-736a1f385c939ce1414e8e279f4e5834-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,742 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,743 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,743 INFO [StoreOpener-736a1f385c939ce1414e8e279f4e5834-1 {}] regionserver.HStore(327): Store=736a1f385c939ce1414e8e279f4e5834/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,744 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,745 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ead0f5940987b7964950b8f83e68ce7d, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:43,746 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,746 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,746 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,747 INFO [StoreOpener-0ae1354f6c79b7fe5dfb500ee1eb508d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,749 INFO [StoreOpener-0ae1354f6c79b7fe5dfb500ee1eb508d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ae1354f6c79b7fe5dfb500ee1eb508d columnFamilyName family 2024-11-20T17:04:43,749 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,749 DEBUG [StoreOpener-0ae1354f6c79b7fe5dfb500ee1eb508d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,750 INFO [StoreOpener-0ae1354f6c79b7fe5dfb500ee1eb508d-1 {}] regionserver.HStore(327): Store=0ae1354f6c79b7fe5dfb500ee1eb508d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,750 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,751 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=7 2024-11-20T17:04:43,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b1a96a3d9afa0a1d03650e42aa3e2fd, server=b60eac9bda43,36999,1732122280556 in 190 msec 2024-11-20T17:04:43,752 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,752 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,753 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,755 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/736a1f385c939ce1414e8e279f4e5834/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9b1a96a3d9afa0a1d03650e42aa3e2fd, ASSIGN in 427 msec 2024-11-20T17:04:43,756 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,756 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 736a1f385c939ce1414e8e279f4e5834; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61304559, jitterRate=-0.08649088442325592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,757 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:43,757 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 736a1f385c939ce1414e8e279f4e5834: Running coprocessor pre-open hook at 1732122283738Writing region info on filesystem at 1732122283738Initializing all the Stores at 1732122283740 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283740Cleaning up temporary data from old regions at 1732122283746 (+6 ms)Running coprocessor post-open hooks at 1732122283757 (+11 ms)Region opened successfully at 1732122283757 2024-11-20T17:04:43,759 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:43,759 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => d48d4832d6d33017d258797576aabdc0, NAME => 'test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-20T17:04:43,760 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834., pid=40, masterSystemTime=1732122283658 2024-11-20T17:04:43,760 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,760 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,760 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,760 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,763 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:43,763 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:43,763 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:43,763 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/0ae1354f6c79b7fe5dfb500ee1eb508d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,763 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 17a1952b1af83907647c5d82cf786cc3, NAME => 'test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-20T17:04:43,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=11 2024-11-20T17:04:43,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure ead0f5940987b7964950b8f83e68ce7d, server=b60eac9bda43,36645,1732122280488 in 138 msec 2024-11-20T17:04:43,764 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,764 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,764 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,764 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,764 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 0ae1354f6c79b7fe5dfb500ee1eb508d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59627627, jitterRate=-0.11147911846637726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,764 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:43,764 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 0ae1354f6c79b7fe5dfb500ee1eb508d: Running coprocessor pre-open hook at 1732122283743Writing region info on filesystem at 1732122283743Initializing all the Stores at 1732122283744 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283744Cleaning up temporary data from old regions at 1732122283753 (+9 ms)Running coprocessor post-open hooks at 1732122283764 (+11 ms)Region opened successfully at 1732122283764 2024-11-20T17:04:43,765 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d., pid=48, masterSystemTime=1732122283681 2024-11-20T17:04:43,766 INFO [StoreOpener-17a1952b1af83907647c5d82cf786cc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ead0f5940987b7964950b8f83e68ce7d, ASSIGN in 439 msec 2024-11-20T17:04:43,769 INFO [StoreOpener-17a1952b1af83907647c5d82cf786cc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17a1952b1af83907647c5d82cf786cc3 columnFamilyName family 2024-11-20T17:04:43,769 DEBUG [StoreOpener-17a1952b1af83907647c5d82cf786cc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,770 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:43,770 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:43,770 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:43,770 INFO [StoreOpener-17a1952b1af83907647c5d82cf786cc3-1 {}] regionserver.HStore(327): Store=17a1952b1af83907647c5d82cf786cc3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,771 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => be724bb9504bda880d6a757168537eb1, NAME => 'test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-20T17:04:43,771 INFO [StoreOpener-d48d4832d6d33017d258797576aabdc0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,771 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,771 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,771 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,771 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,771 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,772 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,773 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,774 INFO [StoreOpener-d48d4832d6d33017d258797576aabdc0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d48d4832d6d33017d258797576aabdc0 columnFamilyName family 2024-11-20T17:04:43,774 DEBUG [StoreOpener-d48d4832d6d33017d258797576aabdc0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,775 INFO [StoreOpener-d48d4832d6d33017d258797576aabdc0-1 {}] regionserver.HStore(327): Store=d48d4832d6d33017d258797576aabdc0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,775 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,776 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,776 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,777 INFO [StoreOpener-be724bb9504bda880d6a757168537eb1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,777 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,777 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,777 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,779 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,779 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,780 INFO [StoreOpener-be724bb9504bda880d6a757168537eb1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be724bb9504bda880d6a757168537eb1 columnFamilyName family 2024-11-20T17:04:43,780 DEBUG [StoreOpener-be724bb9504bda880d6a757168537eb1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,782 INFO [StoreOpener-be724bb9504bda880d6a757168537eb1-1 {}] regionserver.HStore(327): Store=be724bb9504bda880d6a757168537eb1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 736a1f385c939ce1414e8e279f4e5834, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:43,782 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,783 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,784 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,784 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,785 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,785 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,785 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ae1354f6c79b7fe5dfb500ee1eb508d, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:43,789 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=13 2024-11-20T17:04:43,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 736a1f385c939ce1414e8e279f4e5834, server=b60eac9bda43,36999,1732122280556 in 223 msec 2024-11-20T17:04:43,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=736a1f385c939ce1414e8e279f4e5834, ASSIGN in 467 msec 2024-11-20T17:04:43,795 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d48d4832d6d33017d258797576aabdc0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,796 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/17a1952b1af83907647c5d82cf786cc3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,796 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 17a1952b1af83907647c5d82cf786cc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71873902, jitterRate=0.07100459933280945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,797 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:43,796 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened d48d4832d6d33017d258797576aabdc0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67205453, jitterRate=0.0014392882585525513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,797 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:43,797 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for d48d4832d6d33017d258797576aabdc0: Running coprocessor pre-open hook at 1732122283766Writing region info on filesystem at 1732122283766Initializing all the Stores at 1732122283768 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283768Cleaning up temporary data from old regions at 1732122283779 (+11 ms)Running coprocessor post-open hooks at 1732122283797 (+18 ms)Region opened successfully at 1732122283797 2024-11-20T17:04:43,797 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 17a1952b1af83907647c5d82cf786cc3: Running coprocessor pre-open hook at 1732122283764Writing region info on filesystem at 1732122283764Initializing all the Stores at 1732122283765 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283766 (+1 ms)Cleaning up temporary data from old regions at 1732122283777 (+11 ms)Running coprocessor post-open hooks at 1732122283797 (+20 ms)Region opened successfully at 1732122283797 2024-11-20T17:04:43,798 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3., pid=31, masterSystemTime=1732122283658 2024-11-20T17:04:43,798 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0., pid=54, masterSystemTime=1732122283750 2024-11-20T17:04:43,800 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/be724bb9504bda880d6a757168537eb1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=6 2024-11-20T17:04:43,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ae1354f6c79b7fe5dfb500ee1eb508d, server=b60eac9bda43,36645,1732122280488 in 183 msec 2024-11-20T17:04:43,802 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:43,802 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:43,802 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. 2024-11-20T17:04:43,802 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened be724bb9504bda880d6a757168537eb1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67273325, jitterRate=0.0024506598711013794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,803 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:43,803 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => d01fc4378925287cb15b42165d6247f2, NAME => 'test,www,1732122282892.d01fc4378925287cb15b42165d6247f2.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-20T17:04:43,803 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for be724bb9504bda880d6a757168537eb1: Running coprocessor pre-open hook at 1732122283772Writing region info on filesystem at 1732122283772Initializing all the Stores at 1732122283776 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283776Cleaning up temporary data from old regions at 1732122283785 (+9 ms)Running coprocessor post-open hooks at 1732122283803 (+18 ms)Region opened successfully at 1732122283803 2024-11-20T17:04:43,803 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,803 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,www,1732122282892.d01fc4378925287cb15b42165d6247f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,803 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,803 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,804 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1., pid=44, masterSystemTime=1732122283681 2024-11-20T17:04:43,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0ae1354f6c79b7fe5dfb500ee1eb508d, ASSIGN in 476 msec 2024-11-20T17:04:43,806 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:43,806 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:43,806 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:43,806 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 7d0e6a5001fbe78b88c20912afb21151, NAME => 'test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-20T17:04:43,807 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,807 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,807 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,807 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,807 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,808 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:43,808 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:43,808 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:43,808 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => c41b1abb64df3eda0e1232a964811d27, NAME => 'test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-20T17:04:43,809 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,809 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,809 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,809 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,809 INFO [StoreOpener-d01fc4378925287cb15b42165d6247f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17a1952b1af83907647c5d82cf786cc3, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:43,812 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=be724bb9504bda880d6a757168537eb1, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure be724bb9504bda880d6a757168537eb1, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:43,820 INFO [StoreOpener-7d0e6a5001fbe78b88c20912afb21151-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-20T17:04:43,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 17a1952b1af83907647c5d82cf786cc3, server=b60eac9bda43,36999,1732122280556 in 313 msec 2024-11-20T17:04:43,821 INFO [StoreOpener-c41b1abb64df3eda0e1232a964811d27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,821 INFO [StoreOpener-d01fc4378925287cb15b42165d6247f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d01fc4378925287cb15b42165d6247f2 columnFamilyName family 2024-11-20T17:04:43,821 DEBUG [StoreOpener-d01fc4378925287cb15b42165d6247f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,822 INFO [StoreOpener-d01fc4378925287cb15b42165d6247f2-1 {}] regionserver.HStore(327): Store=d01fc4378925287cb15b42165d6247f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,822 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,822 INFO [StoreOpener-7d0e6a5001fbe78b88c20912afb21151-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d0e6a5001fbe78b88c20912afb21151 columnFamilyName family 2024-11-20T17:04:43,823 DEBUG [StoreOpener-7d0e6a5001fbe78b88c20912afb21151-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,823 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,823 INFO [StoreOpener-7d0e6a5001fbe78b88c20912afb21151-1 {}] regionserver.HStore(327): Store=7d0e6a5001fbe78b88c20912afb21151/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,824 INFO [StoreOpener-c41b1abb64df3eda0e1232a964811d27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c41b1abb64df3eda0e1232a964811d27 columnFamilyName family 2024-11-20T17:04:43,824 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,824 DEBUG [StoreOpener-c41b1abb64df3eda0e1232a964811d27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,824 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,825 INFO [StoreOpener-c41b1abb64df3eda0e1232a964811d27-1 {}] regionserver.HStore(327): Store=c41b1abb64df3eda0e1232a964811d27/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,825 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=17a1952b1af83907647c5d82cf786cc3, ASSIGN in 495 msec 2024-11-20T17:04:43,826 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,826 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,826 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,826 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,827 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,827 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,827 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,827 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,827 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,828 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,829 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,830 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,831 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,834 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c41b1abb64df3eda0e1232a964811d27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,835 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened c41b1abb64df3eda0e1232a964811d27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74033502, jitterRate=0.10318514704704285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,835 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:43,835 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for c41b1abb64df3eda0e1232a964811d27: Running coprocessor pre-open hook at 1732122283810Writing region info on filesystem at 1732122283810Initializing all the Stores at 1732122283820 (+10 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283820Cleaning up temporary data from old regions at 1732122283827 (+7 ms)Running coprocessor post-open hooks at 1732122283835 (+8 ms)Region opened successfully at 1732122283835 2024-11-20T17:04:43,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=14 2024-11-20T17:04:43,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure be724bb9504bda880d6a757168537eb1, server=b60eac9bda43,36645,1732122280488 in 240 msec 2024-11-20T17:04:43,836 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27., pid=34, masterSystemTime=1732122283681 2024-11-20T17:04:43,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=be724bb9504bda880d6a757168537eb1, ASSIGN in 510 msec 2024-11-20T17:04:43,839 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:43,839 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:43,840 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:43,840 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => ad24e48b8f4e256db964da467a470ef8, NAME => 'test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-20T17:04:43,840 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,840 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,841 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,841 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,841 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,843 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d01fc4378925287cb15b42165d6247f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c41b1abb64df3eda0e1232a964811d27, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:43,845 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7d0e6a5001fbe78b88c20912afb21151/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,845 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened d01fc4378925287cb15b42165d6247f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73964620, jitterRate=0.10215872526168823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,845 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:43,845 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=d48d4832d6d33017d258797576aabdc0, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,845 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for d01fc4378925287cb15b42165d6247f2: Running coprocessor pre-open hook at 1732122283804Writing region info on filesystem at 1732122283804Initializing all the Stores at 1732122283806 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283806Cleaning up temporary data from old regions at 1732122283826 (+20 ms)Running coprocessor post-open hooks at 1732122283845 (+19 ms)Region opened successfully at 1732122283845 2024-11-20T17:04:43,846 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 7d0e6a5001fbe78b88c20912afb21151; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73368101, jitterRate=0.09326989948749542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,846 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:43,846 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 7d0e6a5001fbe78b88c20912afb21151: Running coprocessor pre-open hook at 1732122283807Writing region info on filesystem at 1732122283807Initializing all the Stores at 1732122283819 (+12 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283819Cleaning up temporary data from old regions at 1732122283828 (+9 ms)Running coprocessor post-open hooks at 1732122283846 (+18 ms)Region opened successfully at 1732122283846 2024-11-20T17:04:43,847 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151., pid=43, masterSystemTime=1732122283658 2024-11-20T17:04:43,848 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732122282892.d01fc4378925287cb15b42165d6247f2., pid=47, masterSystemTime=1732122283750 2024-11-20T17:04:43,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure d48d4832d6d33017d258797576aabdc0, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:43,850 INFO [StoreOpener-ad24e48b8f4e256db964da467a470ef8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,851 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:43,852 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:43,852 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:43,852 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 9de6e6c862796cabf8368e339eb39e67, NAME => 'test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-20T17:04:43,853 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,853 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. 2024-11-20T17:04:43,853 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. 2024-11-20T17:04:43,853 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,853 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,853 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b. 2024-11-20T17:04:43,853 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,853 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => b830d024dd19ff173c226582a1d0e05b, NAME => 'test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-20T17:04:43,854 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,854 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,854 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,854 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,855 INFO [StoreOpener-ad24e48b8f4e256db964da467a470ef8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad24e48b8f4e256db964da467a470ef8 columnFamilyName family 2024-11-20T17:04:43,855 DEBUG [StoreOpener-ad24e48b8f4e256db964da467a470ef8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=30 2024-11-20T17:04:43,856 INFO [StoreOpener-9de6e6c862796cabf8368e339eb39e67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure c41b1abb64df3eda0e1232a964811d27, server=b60eac9bda43,36645,1732122280488 in 323 msec 2024-11-20T17:04:43,856 INFO [StoreOpener-ad24e48b8f4e256db964da467a470ef8-1 {}] regionserver.HStore(327): Store=ad24e48b8f4e256db964da467a470ef8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,856 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,858 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,858 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,858 INFO [StoreOpener-b830d024dd19ff173c226582a1d0e05b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,859 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,859 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,859 INFO [StoreOpener-9de6e6c862796cabf8368e339eb39e67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9de6e6c862796cabf8368e339eb39e67 columnFamilyName family 2024-11-20T17:04:43,859 DEBUG [StoreOpener-9de6e6c862796cabf8368e339eb39e67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,861 INFO [StoreOpener-9de6e6c862796cabf8368e339eb39e67-1 {}] regionserver.HStore(327): Store=9de6e6c862796cabf8368e339eb39e67/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,861 INFO [StoreOpener-b830d024dd19ff173c226582a1d0e05b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b830d024dd19ff173c226582a1d0e05b columnFamilyName family 2024-11-20T17:04:43,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c41b1abb64df3eda0e1232a964811d27, ASSIGN in 531 msec 2024-11-20T17:04:43,861 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,861 DEBUG [StoreOpener-b830d024dd19ff173c226582a1d0e05b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,862 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,863 INFO [StoreOpener-b830d024dd19ff173c226582a1d0e05b-1 {}] regionserver.HStore(327): Store=b830d024dd19ff173c226582a1d0e05b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,863 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d01fc4378925287cb15b42165d6247f2, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,864 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,866 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,866 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,867 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,867 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,867 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,868 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,868 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,868 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/ad24e48b8f4e256db964da467a470ef8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:43,869 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened ad24e48b8f4e256db964da467a470ef8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70782697, jitterRate=0.054744377732276917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,869 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:43,869 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for ad24e48b8f4e256db964da467a470ef8: Running coprocessor pre-open hook at 1732122283841Writing region info on filesystem at 1732122283841Initializing all the Stores at 1732122283843 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283843Cleaning up temporary data from old regions at 1732122283859 (+16 ms)Running coprocessor post-open hooks at 1732122283869 (+10 ms)Region opened successfully at 1732122283869 2024-11-20T17:04:43,870 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,872 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,874 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8., pid=35, masterSystemTime=1732122283681 2024-11-20T17:04:43,874 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/9de6e6c862796cabf8368e339eb39e67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,876 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 9de6e6c862796cabf8368e339eb39e67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63694170, jitterRate=-0.05088290572166443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,876 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:43,876 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 9de6e6c862796cabf8368e339eb39e67: Running coprocessor pre-open hook at 1732122283853Writing region info on filesystem at 1732122283853Initializing all the Stores at 1732122283855 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283855Cleaning up temporary data from old regions at 1732122283867 (+12 ms)Running coprocessor post-open hooks at 1732122283876 (+9 ms)Region opened successfully at 1732122283876 2024-11-20T17:04:43,877 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67., pid=42, masterSystemTime=1732122283658 2024-11-20T17:04:43,878 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/b830d024dd19ff173c226582a1d0e05b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,879 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened b830d024dd19ff173c226582a1d0e05b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63447292, jitterRate=-0.05456167459487915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,879 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b830d024dd19ff173c226582a1d0e05b 2024-11-20T17:04:43,879 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for b830d024dd19ff173c226582a1d0e05b: Running coprocessor pre-open hook at 1732122283854Writing region info on filesystem at 1732122283854Initializing all the Stores at 1732122283856 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283856Cleaning up temporary data from old regions at 1732122283868 (+12 ms)Running coprocessor post-open hooks at 1732122283879 (+11 ms)Region opened successfully at 1732122283879 2024-11-20T17:04:43,880 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:43,880 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:43,880 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:43,881 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => c7bdef0be62b91bb7a8d2486eee55f68, NAME => 'test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-20T17:04:43,881 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:43,881 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:43,881 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,881 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:43,881 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=27 2024-11-20T17:04:43,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,34957,1732122280382 in 268 msec 2024-11-20T17:04:43,881 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,881 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 12d417e0d4f0edac4d7bcadfbf305d96, NAME => 'test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-20T17:04:43,881 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,882 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,882 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,882 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,882 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,882 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b., pid=51, masterSystemTime=1732122283750 2024-11-20T17:04:43,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN in 556 msec 2024-11-20T17:04:43,886 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b. 2024-11-20T17:04:43,886 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,qqq,1732122282892.b830d024dd19ff173c226582a1d0e05b. 2024-11-20T17:04:43,886 INFO [StoreOpener-12d417e0d4f0edac4d7bcadfbf305d96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,886 INFO [StoreOpener-c7bdef0be62b91bb7a8d2486eee55f68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,886 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:43,886 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b8c248055134523e6b12d02a033d557, NAME => 'test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-20T17:04:43,886 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,887 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,887 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,887 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,888 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,888 INFO [StoreOpener-c7bdef0be62b91bb7a8d2486eee55f68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7bdef0be62b91bb7a8d2486eee55f68 columnFamilyName family 2024-11-20T17:04:43,888 INFO [StoreOpener-12d417e0d4f0edac4d7bcadfbf305d96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12d417e0d4f0edac4d7bcadfbf305d96 columnFamilyName family 2024-11-20T17:04:43,888 DEBUG [StoreOpener-c7bdef0be62b91bb7a8d2486eee55f68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,888 DEBUG [StoreOpener-12d417e0d4f0edac4d7bcadfbf305d96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,889 INFO [StoreOpener-c7bdef0be62b91bb7a8d2486eee55f68-1 {}] regionserver.HStore(327): Store=c7bdef0be62b91bb7a8d2486eee55f68/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,889 INFO [StoreOpener-12d417e0d4f0edac4d7bcadfbf305d96-1 {}] regionserver.HStore(327): Store=12d417e0d4f0edac4d7bcadfbf305d96/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,889 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,889 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,891 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,891 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,892 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,892 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,892 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,892 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,892 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,892 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9de6e6c862796cabf8368e339eb39e67, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:43,895 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b60eac9bda43,34957,1732122280382, table=test, region=d48d4832d6d33017d258797576aabdc0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T17:04:43,896 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=b830d024dd19ff173c226582a1d0e05b, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,896 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,896 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:43,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=22 2024-11-20T17:04:43,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 9de6e6c862796cabf8368e339eb39e67, server=b60eac9bda43,36999,1732122280556 in 327 msec 2024-11-20T17:04:43,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9de6e6c862796cabf8368e339eb39e67, ASSIGN in 579 msec 2024-11-20T17:04:43,907 INFO [StoreOpener-7b8c248055134523e6b12d02a033d557-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,908 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/12d417e0d4f0edac4d7bcadfbf305d96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,908 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 12d417e0d4f0edac4d7bcadfbf305d96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73262386, jitterRate=0.09169462323188782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,909 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:43,909 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 12d417e0d4f0edac4d7bcadfbf305d96: Running coprocessor pre-open hook at 1732122283882Writing region info on filesystem at 1732122283882Initializing all the Stores at 1732122283885 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283885Cleaning up temporary data from old regions at 1732122283892 (+7 ms)Running coprocessor post-open hooks at 1732122283909 (+17 ms)Region opened successfully at 1732122283909 2024-11-20T17:04:43,909 INFO [StoreOpener-7b8c248055134523e6b12d02a033d557-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b8c248055134523e6b12d02a033d557 columnFamilyName family 2024-11-20T17:04:43,909 DEBUG [StoreOpener-7b8c248055134523e6b12d02a033d557-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,910 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96., pid=33, masterSystemTime=1732122283658 2024-11-20T17:04:43,910 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c7bdef0be62b91bb7a8d2486eee55f68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,910 INFO [StoreOpener-7b8c248055134523e6b12d02a033d557-1 {}] regionserver.HStore(327): Store=7b8c248055134523e6b12d02a033d557/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,910 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,911 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened c7bdef0be62b91bb7a8d2486eee55f68; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72534109, jitterRate=0.08084245026111603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,911 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:43,911 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for c7bdef0be62b91bb7a8d2486eee55f68: Running coprocessor pre-open hook at 1732122283882Writing region info on filesystem at 1732122283882Initializing all the Stores at 1732122283884 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283884Cleaning up temporary data from old regions at 1732122283892 (+8 ms)Running coprocessor post-open hooks at 1732122283911 (+19 ms)Region opened successfully at 1732122283911 2024-11-20T17:04:43,912 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68., pid=38, masterSystemTime=1732122283681 2024-11-20T17:04:43,912 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,914 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,915 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:43,915 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:43,915 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,915 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=21 2024-11-20T17:04:43,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,34957,1732122280382 in 279 msec 2024-11-20T17:04:43,916 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:43,916 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:43,917 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:43,917 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => d9dc92a1bd5b9a54b65f86671af0015f, NAME => 'test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-20T17:04:43,917 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,917 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,918 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,918 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,918 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN in 591 msec 2024-11-20T17:04:43,920 INFO [StoreOpener-d9dc92a1bd5b9a54b65f86671af0015f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,923 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7b8c248055134523e6b12d02a033d557/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,923 INFO [StoreOpener-d9dc92a1bd5b9a54b65f86671af0015f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9dc92a1bd5b9a54b65f86671af0015f columnFamilyName family 2024-11-20T17:04:43,923 DEBUG [StoreOpener-d9dc92a1bd5b9a54b65f86671af0015f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,924 INFO [StoreOpener-d9dc92a1bd5b9a54b65f86671af0015f-1 {}] regionserver.HStore(327): Store=d9dc92a1bd5b9a54b65f86671af0015f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,924 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,924 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 7b8c248055134523e6b12d02a033d557; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71343269, jitterRate=0.06309755146503448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,924 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:43,924 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 7b8c248055134523e6b12d02a033d557: Running coprocessor pre-open hook at 1732122283887Writing region info on filesystem at 1732122283887Initializing all the Stores at 1732122283888 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283888Cleaning up temporary data from old regions at 1732122283915 (+27 ms)Running coprocessor post-open hooks at 1732122283924 (+9 ms)Region opened successfully at 1732122283924 2024-11-20T17:04:43,925 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,926 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557., pid=46, masterSystemTime=1732122283750 2024-11-20T17:04:43,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7bdef0be62b91bb7a8d2486eee55f68, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:43,927 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,928 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,928 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,929 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:43,929 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:43,930 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560. 2024-11-20T17:04:43,930 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b4bd07e3663cc64a25dde8cd5845560, NAME => 'test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-20T17:04:43,930 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,930 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,930 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,930 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,931 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,933 INFO [StoreOpener-5b4bd07e3663cc64a25dde8cd5845560-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=26 2024-11-20T17:04:43,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure c7bdef0be62b91bb7a8d2486eee55f68, server=b60eac9bda43,36645,1732122280488 in 378 msec 2024-11-20T17:04:43,935 INFO [StoreOpener-5b4bd07e3663cc64a25dde8cd5845560-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b4bd07e3663cc64a25dde8cd5845560 columnFamilyName family 2024-11-20T17:04:43,935 DEBUG [StoreOpener-5b4bd07e3663cc64a25dde8cd5845560-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,936 INFO [StoreOpener-5b4bd07e3663cc64a25dde8cd5845560-1 {}] regionserver.HStore(327): Store=5b4bd07e3663cc64a25dde8cd5845560/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,937 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c7bdef0be62b91bb7a8d2486eee55f68, ASSIGN in 610 msec 2024-11-20T17:04:43,938 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,938 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,939 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d9dc92a1bd5b9a54b65f86671af0015f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,939 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,939 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,939 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened d9dc92a1bd5b9a54b65f86671af0015f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73416742, jitterRate=0.09399470686912537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,940 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=7b8c248055134523e6b12d02a033d557, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,940 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:43,940 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for d9dc92a1bd5b9a54b65f86671af0015f: Running coprocessor pre-open hook at 1732122283918Writing region info on filesystem at 1732122283918Initializing all the Stores at 1732122283920 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283920Cleaning up temporary data from old regions at 1732122283928 (+8 ms)Running coprocessor post-open hooks at 1732122283940 (+12 ms)Region opened successfully at 1732122283940 2024-11-20T17:04:43,942 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f., pid=36, masterSystemTime=1732122283681 2024-11-20T17:04:43,943 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:43,944 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:43,947 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:43,947 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:43,947 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:43,947 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => e0f5627de02ae3387d78a84cf60de116, NAME => 'test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-20T17:04:43,948 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12d417e0d4f0edac4d7bcadfbf305d96, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:43,948 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,948 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/5b4bd07e3663cc64a25dde8cd5845560/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,948 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,948 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,949 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 5b4bd07e3663cc64a25dde8cd5845560; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74147697, jitterRate=0.10488678514957428}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,949 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b4bd07e3663cc64a25dde8cd5845560 2024-11-20T17:04:43,949 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 5b4bd07e3663cc64a25dde8cd5845560: Running coprocessor pre-open hook at 1732122283931Writing region info on filesystem at 1732122283931Initializing all the Stores at 1732122283932 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283932Cleaning up temporary data from old regions at 1732122283940 (+8 ms)Running coprocessor post-open hooks at 1732122283949 (+9 ms)Region opened successfully at 1732122283949 2024-11-20T17:04:43,950 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560., pid=53, masterSystemTime=1732122283750 2024-11-20T17:04:43,950 INFO [StoreOpener-e0f5627de02ae3387d78a84cf60de116-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,953 INFO [StoreOpener-e0f5627de02ae3387d78a84cf60de116-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0f5627de02ae3387d78a84cf60de116 columnFamilyName family 2024-11-20T17:04:43,953 DEBUG [StoreOpener-e0f5627de02ae3387d78a84cf60de116-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,953 INFO [StoreOpener-e0f5627de02ae3387d78a84cf60de116-1 {}] regionserver.HStore(327): Store=e0f5627de02ae3387d78a84cf60de116/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,954 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,954 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560. 2024-11-20T17:04:43,954 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560. 2024-11-20T17:04:43,954 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:43,955 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 052ad436a3fbddb02bb8de2342c3b9a7, NAME => 'test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-20T17:04:43,955 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,955 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,955 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,955 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,956 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-11-20T17:04:43,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 12d417e0d4f0edac4d7bcadfbf305d96, server=b60eac9bda43,36999,1732122280556 in 431 msec 2024-11-20T17:04:43,957 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,957 INFO [StoreOpener-052ad436a3fbddb02bb8de2342c3b9a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,958 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,958 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=12d417e0d4f0edac4d7bcadfbf305d96, ASSIGN in 631 msec 2024-11-20T17:04:43,960 INFO [StoreOpener-052ad436a3fbddb02bb8de2342c3b9a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 052ad436a3fbddb02bb8de2342c3b9a7 columnFamilyName family 2024-11-20T17:04:43,960 DEBUG [StoreOpener-052ad436a3fbddb02bb8de2342c3b9a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,961 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,961 INFO [StoreOpener-052ad436a3fbddb02bb8de2342c3b9a7-1 {}] regionserver.HStore(327): Store=052ad436a3fbddb02bb8de2342c3b9a7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,961 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,962 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,962 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,963 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,964 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,964 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,965 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/e0f5627de02ae3387d78a84cf60de116/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,966 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened e0f5627de02ae3387d78a84cf60de116; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60983409, jitterRate=-0.0912763923406601}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,966 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:43,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:43,966 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,966 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for e0f5627de02ae3387d78a84cf60de116: Running coprocessor pre-open hook at 1732122283948Writing region info on filesystem at 1732122283948Initializing all the Stores at 1732122283949 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283949Cleaning up temporary data from old regions at 1732122283958 (+9 ms)Running coprocessor post-open hooks at 1732122283966 (+8 ms)Region opened successfully at 1732122283966 2024-11-20T17:04:43,968 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116., pid=49, masterSystemTime=1732122283681 2024-11-20T17:04:43,971 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9dc92a1bd5b9a54b65f86671af0015f, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:43,973 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b60eac9bda43,36645,1732122280488, table=test, region=d9dc92a1bd5b9a54b65f86671af0015f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T17:04:43,974 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:43,974 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:43,976 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/052ad436a3fbddb02bb8de2342c3b9a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:43,977 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 052ad436a3fbddb02bb8de2342c3b9a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72577589, jitterRate=0.08149035274982452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:43,977 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:43,977 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 052ad436a3fbddb02bb8de2342c3b9a7: Running coprocessor pre-open hook at 1732122283956Writing region info on filesystem at 1732122283956Initializing all the Stores at 1732122283957 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283957Cleaning up temporary data from old regions at 1732122283964 (+7 ms)Running coprocessor post-open hooks at 1732122283977 (+13 ms)Region opened successfully at 1732122283977 2024-11-20T17:04:43,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=9 2024-11-20T17:04:43,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,34957,1732122280382 in 335 msec 2024-11-20T17:04:43,979 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7., pid=55, masterSystemTime=1732122283750 2024-11-20T17:04:43,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN in 653 msec 2024-11-20T17:04:43,981 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:43,981 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:43,982 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. 2024-11-20T17:04:43,982 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => a7a9afb4414c2ce282ecb8a226b732ef, NAME => 'test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-20T17:04:43,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:43,982 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,982 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:43,983 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,983 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,985 INFO [StoreOpener-a7a9afb4414c2ce282ecb8a226b732ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:43,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:43,988 INFO [StoreOpener-a7a9afb4414c2ce282ecb8a226b732ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7a9afb4414c2ce282ecb8a226b732ef columnFamilyName family 2024-11-20T17:04:43,988 DEBUG [StoreOpener-a7a9afb4414c2ce282ecb8a226b732ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:43,989 INFO [StoreOpener-a7a9afb4414c2ce282ecb8a226b732ef-1 {}] regionserver.HStore(327): Store=a7a9afb4414c2ce282ecb8a226b732ef/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:43,989 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0f5627de02ae3387d78a84cf60de116, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:43,991 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,991 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,992 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,992 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=18 2024-11-20T17:04:43,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,34957,1732122280382 in 346 msec 2024-11-20T17:04:43,994 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:43,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN in 668 msec 2024-11-20T17:04:43,999 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b60eac9bda43,34957,1732122280382, table=test, region=d48d4832d6d33017d258797576aabdc0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T17:04:43,999 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b60eac9bda43,34957,1732122280382, table=test, region=7b8c248055134523e6b12d02a033d557. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T17:04:44,001 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/a7a9afb4414c2ce282ecb8a226b732ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:44,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=8 2024-11-20T17:04:44,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e0f5627de02ae3387d78a84cf60de116, server=b60eac9bda43,36645,1732122280488 in 381 msec 2024-11-20T17:04:44,001 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened a7a9afb4414c2ce282ecb8a226b732ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70485940, jitterRate=0.05032235383987427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:44,002 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:44,002 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for a7a9afb4414c2ce282ecb8a226b732ef: Running coprocessor pre-open hook at 1732122283983Writing region info on filesystem at 1732122283983Initializing all the Stores at 1732122283985 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122283985Cleaning up temporary data from old regions at 1732122283992 (+7 ms)Running coprocessor post-open hooks at 1732122284002 (+10 ms)Region opened successfully at 1732122284002 2024-11-20T17:04:44,003 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef., pid=56, masterSystemTime=1732122283750 2024-11-20T17:04:44,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e0f5627de02ae3387d78a84cf60de116, ASSIGN in 676 msec 2024-11-20T17:04:44,006 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. 2024-11-20T17:04:44,006 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. 2024-11-20T17:04:44,006 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39. 2024-11-20T17:04:44,006 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e0f80ef67003cc41131beab8b406f39, NAME => 'test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-20T17:04:44,007 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,007 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:44,007 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,007 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=17 2024-11-20T17:04:44,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d9dc92a1bd5b9a54b65f86671af0015f, server=b60eac9bda43,36645,1732122280488 in 466 msec 2024-11-20T17:04:44,010 INFO [StoreOpener-0e0f80ef67003cc41131beab8b406f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d9dc92a1bd5b9a54b65f86671af0015f, ASSIGN in 684 msec 2024-11-20T17:04:44,013 INFO [StoreOpener-0e0f80ef67003cc41131beab8b406f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e0f80ef67003cc41131beab8b406f39 columnFamilyName family 2024-11-20T17:04:44,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:44,013 DEBUG [StoreOpener-0e0f80ef67003cc41131beab8b406f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:44,014 INFO [StoreOpener-0e0f80ef67003cc41131beab8b406f39-1 {}] regionserver.HStore(327): Store=0e0f80ef67003cc41131beab8b406f39/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:44,014 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,016 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:44,017 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,018 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,018 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,020 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=24 2024-11-20T17:04:44,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,34957,1732122280382 in 415 msec 2024-11-20T17:04:44,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN in 696 msec 2024-11-20T17:04:44,024 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/0e0f80ef67003cc41131beab8b406f39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:44,024 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 0e0f80ef67003cc41131beab8b406f39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65112472, jitterRate=-0.02974855899810791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:44,025 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e0f80ef67003cc41131beab8b406f39 2024-11-20T17:04:44,025 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 0e0f80ef67003cc41131beab8b406f39: Running coprocessor pre-open hook at 1732122284008Writing region info on filesystem at 1732122284008Initializing all the Stores at 1732122284009 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122284009Cleaning up temporary data from old regions at 1732122284018 (+9 ms)Running coprocessor post-open hooks at 1732122284025 (+7 ms)Region opened successfully at 1732122284025 2024-11-20T17:04:44,026 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39., pid=52, masterSystemTime=1732122283750 2024-11-20T17:04:44,028 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39. 2024-11-20T17:04:44,029 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,lll,1732122282892.0e0f80ef67003cc41131beab8b406f39. 2024-11-20T17:04:44,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=5 2024-11-20T17:04:44,029 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:44,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,34957,1732122280382 in 374 msec 2024-11-20T17:04:44,029 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => fd9711911c68abdd7f4a5f7d15d952e9, NAME => 'test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-20T17:04:44,029 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,029 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:44,030 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,030 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN in 704 msec 2024-11-20T17:04:44,032 INFO [StoreOpener-fd9711911c68abdd7f4a5f7d15d952e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=0e0f80ef67003cc41131beab8b406f39, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:44,034 INFO [StoreOpener-fd9711911c68abdd7f4a5f7d15d952e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd9711911c68abdd7f4a5f7d15d952e9 columnFamilyName family 2024-11-20T17:04:44,034 DEBUG [StoreOpener-fd9711911c68abdd7f4a5f7d15d952e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:44,035 INFO [StoreOpener-fd9711911c68abdd7f4a5f7d15d952e9-1 {}] regionserver.HStore(327): Store=fd9711911c68abdd7f4a5f7d15d952e9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:44,035 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,036 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:44,037 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,038 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,038 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:44,038 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,041 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad24e48b8f4e256db964da467a470ef8, server=b60eac9bda43,36645,1732122280488 because future has completed 2024-11-20T17:04:44,044 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/fd9711911c68abdd7f4a5f7d15d952e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:04:44,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=16 2024-11-20T17:04:44,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,34957,1732122280382 in 409 msec 2024-11-20T17:04:44,046 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened fd9711911c68abdd7f4a5f7d15d952e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60403423, jitterRate=-0.09991885721683502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:44,046 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:44,046 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for fd9711911c68abdd7f4a5f7d15d952e9: Running coprocessor pre-open hook at 1732122284030Writing region info on filesystem at 1732122284030Initializing all the Stores at 1732122284031 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122284031Cleaning up temporary data from old regions at 1732122284038 (+7 ms)Running coprocessor post-open hooks at 1732122284046 (+8 ms)Region opened successfully at 1732122284046 2024-11-20T17:04:44,047 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9., pid=45, masterSystemTime=1732122283750 2024-11-20T17:04:44,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN in 721 msec 2024-11-20T17:04:44,050 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:44,050 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:44,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=20 2024-11-20T17:04:44,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure ad24e48b8f4e256db964da467a470ef8, server=b60eac9bda43,36645,1732122280488 in 515 msec 2024-11-20T17:04:44,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ad24e48b8f4e256db964da467a470ef8, ASSIGN in 727 msec 2024-11-20T17:04:44,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:44,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:44,058 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, openSeqNum=2, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:44,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d0e6a5001fbe78b88c20912afb21151, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:44,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=29 2024-11-20T17:04:44,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,34957,1732122280382 in 465 msec 2024-11-20T17:04:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:04:44,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, ASSIGN in 739 msec 2024-11-20T17:04:44,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=25 2024-11-20T17:04:44,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 7d0e6a5001fbe78b88c20912afb21151, server=b60eac9bda43,36999,1732122280556 in 488 msec 2024-11-20T17:04:44,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7d0e6a5001fbe78b88c20912afb21151, ASSIGN in 744 msec 2024-11-20T17:04:44,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=12 2024-11-20T17:04:44,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure d48d4832d6d33017d258797576aabdc0, server=b60eac9bda43,34957,1732122280382 in 431 msec 2024-11-20T17:04:44,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=4 2024-11-20T17:04:44,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN in 749 msec 2024-11-20T17:04:44,080 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:04:44,080 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732122284080"}]},"ts":"1732122284080"} 2024-11-20T17:04:44,083 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-20T17:04:44,085 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:04:44,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.1860 sec 2024-11-20T17:04:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:04:45,075 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-20T17:04:45,075 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-20T17:04:45,077 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:04:45,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-20T17:04:45,098 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:04:45,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-20T17:04:45,099 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:04:45,104 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T17:04:45,129 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T17:04:45,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:04:45,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:04:45,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:04:45,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:04:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a7a9afb4414c2ce282ecb8a226b732ef changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5b4bd07e3663cc64a25dde8cd5845560 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d48d4832d6d33017d258797576aabdc0 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0e0f80ef67003cc41131beab8b406f39 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 052ad436a3fbddb02bb8de2342c3b9a7 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b830d024dd19ff173c226582a1d0e05b changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7b8c248055134523e6b12d02a033d557 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d01fc4378925287cb15b42165d6247f2 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fd9711911c68abdd7f4a5f7d15d952e9 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0ae1354f6c79b7fe5dfb500ee1eb508d changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e0f5627de02ae3387d78a84cf60de116 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ead0f5940987b7964950b8f83e68ce7d changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region be724bb9504bda880d6a757168537eb1 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d9dc92a1bd5b9a54b65f86671af0015f changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ad24e48b8f4e256db964da467a470ef8 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c22acb23a82c5ea894235ab5c0c16804 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c7bdef0be62b91bb7a8d2486eee55f68 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c41b1abb64df3eda0e1232a964811d27 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9b1a96a3d9afa0a1d03650e42aa3e2fd changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 12d417e0d4f0edac4d7bcadfbf305d96 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 736a1f385c939ce1414e8e279f4e5834 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f5cc1a8141b65dd3101cc2eb478bf7fc changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 13fb0e6117ef98d16b8be8bc4a741d1b changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9de6e6c862796cabf8368e339eb39e67 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7d0e6a5001fbe78b88c20912afb21151 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 17a1952b1af83907647c5d82cf786cc3 changed from -1.0 to 0.0, refreshing cache 2024-11-20T17:04:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-20T17:04:45,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-20T17:04:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-20T17:04:45,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(204): Hosts are {b60eac9bda43=0} racks are {/default-rack=0} 2024-11-20T17:04:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-11-20T17:04:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-20T17:04:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-11-20T17:04:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:04:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:04:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T17:04:45,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:04:45,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:04:45,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T17:04:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-20T17:04:45,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-20T17:04:45,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-20T17:04:45,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 501 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-20T17:04:45,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-20T17:04:45,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.HMaster(2172): balance hri=fd9711911c68abdd7f4a5f7d15d952e9, source=b60eac9bda43,34957,1732122280382, destination=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:45,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:45,721 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=fd9711911c68abdd7f4a5f7d15d952e9, regionState=CLOSING, regionLocation=b60eac9bda43,34957,1732122280382 2024-11-20T17:04:45,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE because future has completed 2024-11-20T17:04:45,724 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:04:45,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,34957,1732122280382}] 2024-11-20T17:04:45,881 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:45,881 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:04:45,882 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing fd9711911c68abdd7f4a5f7d15d952e9, disabling compactions & flushes 2024-11-20T17:04:45,882 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:45,882 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:45,882 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. after waiting 0 ms 2024-11-20T17:04:45,882 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:45,889 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/fd9711911c68abdd7f4a5f7d15d952e9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:45,892 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:45,892 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for fd9711911c68abdd7f4a5f7d15d952e9: Waiting for close lock at 1732122285882Running coprocessor pre-close hooks at 1732122285882Disabling compacts and flushes for region at 1732122285882Disabling writes for close at 1732122285882Writing region close event to WAL at 1732122285882Running coprocessor post-close hooks at 1732122285890 (+8 ms)Closed at 1732122285892 (+2 ms) 2024-11-20T17:04:45,892 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding fd9711911c68abdd7f4a5f7d15d952e9 move to b60eac9bda43,36999,1732122280556 record at close sequenceid=2 2024-11-20T17:04:45,896 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:45,896 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=fd9711911c68abdd7f4a5f7d15d952e9, regionState=CLOSED 2024-11-20T17:04:45,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,34957,1732122280382 because future has completed 2024-11-20T17:04:45,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-20T17:04:45,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,34957,1732122280382 in 176 msec 2024-11-20T17:04:45,904 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE; state=CLOSED, location=b60eac9bda43,36999,1732122280556; forceNewPlan=false, retain=false 2024-11-20T17:04:46,054 INFO [b60eac9bda43:44751 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T17:04:46,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:46,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE because future has completed 2024-11-20T17:04:46,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:46,222 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:46,223 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => fd9711911c68abdd7f4a5f7d15d952e9, NAME => 'test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-20T17:04:46,224 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,224 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:46,224 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,225 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,227 INFO [StoreOpener-fd9711911c68abdd7f4a5f7d15d952e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,228 INFO [StoreOpener-fd9711911c68abdd7f4a5f7d15d952e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd9711911c68abdd7f4a5f7d15d952e9 columnFamilyName family 2024-11-20T17:04:46,228 DEBUG [StoreOpener-fd9711911c68abdd7f4a5f7d15d952e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:46,229 INFO [StoreOpener-fd9711911c68abdd7f4a5f7d15d952e9-1 {}] regionserver.HStore(327): Store=fd9711911c68abdd7f4a5f7d15d952e9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:46,229 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,230 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,232 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,233 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,233 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,236 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,237 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened fd9711911c68abdd7f4a5f7d15d952e9; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59893411, jitterRate=-0.10751862823963165}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:46,237 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:46,237 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for fd9711911c68abdd7f4a5f7d15d952e9: Running coprocessor pre-open hook at 1732122286225Writing region info on filesystem at 1732122286225Initializing all the Stores at 1732122286227 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122286227Cleaning up temporary data from old regions at 1732122286233 (+6 ms)Running coprocessor post-open hooks at 1732122286237 (+4 ms)Region opened successfully at 1732122286237 2024-11-20T17:04:46,238 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9., pid=59, masterSystemTime=1732122286212 2024-11-20T17:04:46,241 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:46,241 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:46,242 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, openSeqNum=5, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:46,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,36999,1732122280556 because future has completed 2024-11-20T17:04:46,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-20T17:04:46,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure fd9711911c68abdd7f4a5f7d15d952e9, server=b60eac9bda43,36999,1732122280556 in 187 msec 2024-11-20T17:04:46,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE in 531 msec 2024-11-20T17:04:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-20T17:04:46,323 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:04:46,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-20T17:04:46,343 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=381, OpenFileDescriptor=644, MaxFileDescriptor=1048576, SystemLoadAverage=175, ProcessCount=11, AvailableMemoryMB=7578 2024-11-20T17:04:46,346 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-20T17:04:46,347 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-20T17:04:46,347 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing b60eac9bda43,34957,1732122280382 2024-11-20T17:04:46,348 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b60eac9bda43,34957,1732122280382: Simulated kill ***** 2024-11-20T17:04:46,348 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T17:04:46,349 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T17:04:46,351 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T17:04:46,352 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T17:04:46,352 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T17:04:46,352 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 332375584 }, "NonHeapMemoryUsage": { "committed": 153944064, "init": 7667712, "max": -1, "used": 151686080 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43" } ] } 2024-11-20T17:04:46,355 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44751 {}] master.MasterRpcServices(700): b60eac9bda43,34957,1732122280382 reported a fatal error: ***** ABORTING region server b60eac9bda43,34957,1732122280382: Simulated kill ***** 2024-11-20T17:04:46,357 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b60eac9bda43,34957,1732122280382' ***** 2024-11-20T17:04:46,357 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-20T17:04:46,357 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:04:46,358 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:04:46,358 INFO [RS:0;b60eac9bda43:34957 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T17:04:46,358 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:04:46,358 INFO [RS:0;b60eac9bda43:34957 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T17:04:46,358 INFO [RS:0;b60eac9bda43:34957 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:04:46,358 INFO [RS:0;b60eac9bda43:34957 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b60eac9bda43:34957. 2024-11-20T17:04:46,358 DEBUG [RS:0;b60eac9bda43:34957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:04:46,363 DEBUG [RS:0;b60eac9bda43:34957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:46,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741836_1012 (size=33749) 2024-11-20T17:04:46,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741836_1012 (size=33749) 2024-11-20T17:04:46,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741836_1012 (size=33749) 2024-11-20T17:04:46,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741835_1011 (size=2769) 2024-11-20T17:04:46,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741835_1011 (size=2769) 2024-11-20T17:04:46,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741835_1011 (size=2769) 2024-11-20T17:04:46,382 DEBUG [RS:0;b60eac9bda43:34957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:46,382 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:04:46,382 INFO [RS:0;b60eac9bda43:34957 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34957 2024-11-20T17:04:46,400 INFO [regionserver/b60eac9bda43:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:04:46,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:04:46,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b60eac9bda43,34957,1732122280382 2024-11-20T17:04:46,439 INFO [RS:0;b60eac9bda43:34957 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:04:46,448 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b60eac9bda43,34957,1732122280382] 2024-11-20T17:04:46,456 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b60eac9bda43,34957,1732122280382 already deleted, retry=false 2024-11-20T17:04:46,456 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b60eac9bda43,34957,1732122280382 on b60eac9bda43,44751,1732122279653 2024-11-20T17:04:46,461 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:46,465 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for b60eac9bda43,34957,1732122280382 (carryingMeta=true) b60eac9bda43,34957,1732122280382/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@605f3abd[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T17:04:46,465 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:46,467 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true, isMeta: true 2024-11-20T17:04:46,470 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting 2024-11-20T17:04:46,472 INFO [PEWorker-1 {}] master.SplitWALManager(105): b60eac9bda43,34957,1732122280382 WAL count=1, meta=true 2024-11-20T17:04:46,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta}] 2024-11-20T17:04:46,481 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:46,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:46,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:04:46,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34957-0x1015a7e0e7a0001, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:04:46,549 INFO [RS:0;b60eac9bda43:34957 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:04:46,549 INFO [RS:0;b60eac9bda43:34957 {}] regionserver.HRegionServer(1031): Exiting; stopping=b60eac9bda43,34957,1732122280382; zookeeper connection closed. 2024-11-20T17:04:46,549 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28a037e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28a037e3 2024-11-20T17:04:46,559 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T17:04:46,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36999 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-20T17:04:46,681 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, size=33.0 K (33749bytes) 2024-11-20T17:04:46,682 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T17:04:46,682 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T17:04:46,683 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta 2024-11-20T17:04:46,689 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta after 4ms 2024-11-20T17:04:46,697 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta took 18ms 2024-11-20T17:04:46,705 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T17:04:46,712 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta.temp 2024-11-20T17:04:46,715 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta.temp 2024-11-20T17:04:46,717 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta so closing down 2024-11-20T17:04:46,717 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T17:04:46,719 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T17:04:46,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741864_1040 (size=33508) 2024-11-20T17:04:46,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741864_1040 (size=33508) 2024-11-20T17:04:46,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741864_1040 (size=33508) 2024-11-20T17:04:46,732 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-20T17:04:46,734 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta.temp to hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-20T17:04:46,738 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-20T17:04:46,739 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, journal: Splitting hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, size=33.0 K (33749bytes) at 1732122286681Creating recovered edits writer path=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta.temp at 1732122286715 (+34 ms)Finishing writing output for hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta so closing down at 1732122286717 (+2 ms)3 split writer threads finished at 1732122286719 (+2 ms)Closed recovered edits writer path=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1732122286732 (+13 ms)Rename recovered edits hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta.temp to hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732122286734 (+2 ms)Processed 62 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732122286738 (+4 ms) 2024-11-20T17:04:46,739 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta 2024-11-20T17:04:46,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-20T17:04:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-20T17:04:46,750 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta to hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs 2024-11-20T17:04:46,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-20T17:04:46,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 in 267 msec 2024-11-20T17:04:46,754 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:46,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T17:04:46,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 in 280 msec 2024-11-20T17:04:46,760 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting dir is empty, no logs to split. 2024-11-20T17:04:46,760 INFO [PEWorker-1 {}] master.SplitWALManager(105): b60eac9bda43,34957,1732122280382 WAL count=0, meta=true 2024-11-20T17:04:46,760 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if b60eac9bda43,34957,1732122280382 WAL splitting is done? wals=0, meta=true 2024-11-20T17:04:46,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T17:04:46,764 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:46,765 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:46,900 INFO [regionserver/b60eac9bda43:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T17:04:46,900 INFO [regionserver/b60eac9bda43:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T17:04:46,916 DEBUG [b60eac9bda43:44751 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T17:04:46,916 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(204): Hosts are {b60eac9bda43=0} racks are {/default-rack=0} 2024-11-20T17:04:46,916 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:04:46,916 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:04:46,916 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:04:46,916 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:04:46,917 INFO [b60eac9bda43:44751 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:04:46,917 INFO [b60eac9bda43:44751 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:04:46,917 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:04:46,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:46,918 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPENING 2024-11-20T17:04:46,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:46,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:46,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:46,938 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:46,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:46,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:46,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:46,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:47,104 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T17:04:47,104 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:04:47,105 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T17:04:47,108 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b60eac9bda43%2C36999%2C1732122280556.meta, suffix=.meta, logDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36999,1732122280556, archiveDir=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs, maxLogs=32 2024-11-20T17:04:47,124 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36999,1732122280556/b60eac9bda43%2C36999%2C1732122280556.meta.1732122287110.meta, exclude list is [], retry=0 2024-11-20T17:04:47,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33493,DS-7004819e-1508-46b4-86e4-acb222d36d37,DISK] 2024-11-20T17:04:47,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-4e1882e0-2569-4b10-92a0-f5eec9f95d96,DISK] 2024-11-20T17:04:47,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41999,DS-db2b7c12-a09c-4d0d-aea8-9fea2ae5a604,DISK] 2024-11-20T17:04:47,129 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,36999,1732122280556/b60eac9bda43%2C36999%2C1732122280556.meta.1732122287110.meta 2024-11-20T17:04:47,130 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37891:37891),(127.0.0.1/127.0.0.1:33029:33029),(127.0.0.1/127.0.0.1:45507:45507)] 2024-11-20T17:04:47,131 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:04:47,131 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T17:04:47,131 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T17:04:47,131 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T17:04:47,131 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T17:04:47,131 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:47,131 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T17:04:47,131 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T17:04:47,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T17:04:47,134 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:04:47,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:47,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:47,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T17:04:47,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:04:47,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:47,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:47,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T17:04:47,137 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:04:47,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:47,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:47,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T17:04:47,139 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:04:47,139 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:47,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:04:47,140 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:04:47,141 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740 2024-11-20T17:04:47,143 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740 2024-11-20T17:04:47,145 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-20T17:04:47,160 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-20T17:04:47,163 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-20T17:04:47,238 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/info/43d0b1679192499b8facc1405f47346b is 138, key is test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d./info:regioninfo/1732122283778/Put/seqid=0 2024-11-20T17:04:47,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741866_1042 (size=36376) 2024-11-20T17:04:47,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741866_1042 (size=36376) 2024-11-20T17:04:47,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741866_1042 (size=36376) 2024-11-20T17:04:47,250 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/info/43d0b1679192499b8facc1405f47346b 2024-11-20T17:04:47,317 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/ns/179544c42c084a30ae70de96c3600cc8 is 43, key is default/ns:d/1732122282701/Put/seqid=0 2024-11-20T17:04:47,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741867_1043 (size=5153) 2024-11-20T17:04:47,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741867_1043 (size=5153) 2024-11-20T17:04:47,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741867_1043 (size=5153) 2024-11-20T17:04:47,328 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/ns/179544c42c084a30ae70de96c3600cc8 2024-11-20T17:04:47,356 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/table/efa64de621944c9e8cfbb9e3a4167f17 is 40, key is test/table:state/1732122284080/Put/seqid=0 2024-11-20T17:04:47,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741868_1044 (size=5165) 2024-11-20T17:04:47,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741868_1044 (size=5165) 2024-11-20T17:04:47,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741868_1044 (size=5165) 2024-11-20T17:04:47,365 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/table/efa64de621944c9e8cfbb9e3a4167f17 2024-11-20T17:04:47,374 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/info/43d0b1679192499b8facc1405f47346b as hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/info/43d0b1679192499b8facc1405f47346b 2024-11-20T17:04:47,383 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/info/43d0b1679192499b8facc1405f47346b, entries=273, sequenceid=64, filesize=35.5 K 2024-11-20T17:04:47,386 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/ns/179544c42c084a30ae70de96c3600cc8 as hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/ns/179544c42c084a30ae70de96c3600cc8 2024-11-20T17:04:47,395 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/ns/179544c42c084a30ae70de96c3600cc8, entries=2, sequenceid=64, filesize=5.0 K 2024-11-20T17:04:47,396 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/table/efa64de621944c9e8cfbb9e3a4167f17 as hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/table/efa64de621944c9e8cfbb9e3a4167f17 2024-11-20T17:04:47,405 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/table/efa64de621944c9e8cfbb9e3a4167f17, entries=2, sequenceid=64, filesize=5.0 K 2024-11-20T17:04:47,410 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 245ms, sequenceid=64, compaction requested=false; wal=null 2024-11-20T17:04:47,411 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T17:04:47,415 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-20T17:04:47,418 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:04:47,418 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:04:47,419 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:04:47,421 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:04:47,424 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-20T17:04:47,425 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74862536, jitterRate=0.11553871631622314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:04:47,426 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T17:04:47,426 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732122287131Writing region info on filesystem at 1732122287132 (+1 ms)Initializing all the Stores at 1732122287133 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122287133Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122287133Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122287133Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732122287133Obtaining lock to block concurrent updates at 1732122287163 (+30 ms)Preparing flush snapshotting stores in 1588230740 at 1732122287163Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732122287166 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732122287166Flushing 1588230740/info: creating writer at 1732122287167 (+1 ms)Flushing 1588230740/info: appending metadata at 1732122287227 (+60 ms)Flushing 1588230740/info: closing flushed file at 1732122287229 (+2 ms)Flushing 1588230740/ns: creating writer at 1732122287300 (+71 ms)Flushing 1588230740/ns: appending metadata at 1732122287316 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732122287316Flushing 1588230740/table: creating writer at 1732122287337 (+21 ms)Flushing 1588230740/table: appending metadata at 1732122287355 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732122287355Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@176b7452: reopening flushed file at 1732122287372 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@178ce0b4: reopening flushed file at 1732122287384 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31be6e42: reopening flushed file at 1732122287395 (+11 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 245ms, sequenceid=64, compaction requested=false; wal=null at 1732122287410 (+15 ms)Cleaning up temporary data from old regions at 1732122287418 (+8 ms)Running coprocessor post-open hooks at 1732122287426 (+8 ms)Region opened successfully at 1732122287426 2024-11-20T17:04:47,427 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732122287091 2024-11-20T17:04:47,430 DEBUG [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T17:04:47,430 INFO [RS_OPEN_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T17:04:47,431 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:47,432 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:47,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:47,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:47,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:47,463 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:47,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:47,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:47,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:47,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-20T17:04:47,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 in 525 msec 2024-11-20T17:04:47,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-20T17:04:47,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 703 msec 2024-11-20T17:04:47,469 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): b60eac9bda43,34957,1732122280382 had 9 regions 2024-11-20T17:04:47,470 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true, isMeta: false 2024-11-20T17:04:47,471 INFO [PEWorker-1 {}] master.SplitWALManager(105): b60eac9bda43,34957,1732122280382 WAL count=1, meta=false 2024-11-20T17:04:47,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056}] 2024-11-20T17:04:47,473 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:47,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:47,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36645 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-20T17:04:47,652 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056, size=2.7 K (2769bytes) 2024-11-20T17:04:47,652 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056 2024-11-20T17:04:47,653 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056 after 1ms 2024-11-20T17:04:47,656 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056 took 4ms 2024-11-20T17:04:47,659 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d48d4832d6d33017d258797576aabdc0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,661 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d01fc4378925287cb15b42165d6247f2: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,664 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b830d024dd19ff173c226582a1d0e05b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,666 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 7b8c248055134523e6b12d02a033d557: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,669 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 5b4bd07e3663cc64a25dde8cd5845560: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,671 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 052ad436a3fbddb02bb8de2342c3b9a7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,673 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a7a9afb4414c2ce282ecb8a226b732ef: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,675 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 0e0f80ef67003cc41131beab8b406f39: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for fd9711911c68abdd7f4a5f7d15d952e9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T17:04:47,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056 so closing down 2024-11-20T17:04:47,678 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T17:04:47,678 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T17:04:47,678 INFO [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 22 ms; skipped=10; WAL=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-11-20T17:04:47,678 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056, journal: Splitting hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056, size=2.7 K (2769bytes) at 1732122287652Finishing writing output for hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056 so closing down at 1732122287677 (+25 ms)3 split writer threads finished at 1732122287678 (+1 ms)Processed 10 edits across 0 Regions in 22 ms; skipped=10; WAL=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1732122287678 2024-11-20T17:04:47,678 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056 2024-11-20T17:04:47,678 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b60eac9bda43:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-20T17:04:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44751 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-20T17:04:47,683 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting/b60eac9bda43%2C34957%2C1732122280382.1732122282056 to hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs 2024-11-20T17:04:47,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-20T17:04:47,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 in 209 msec 2024-11-20T17:04:47,688 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:47,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-20T17:04:47,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 in 217 msec 2024-11-20T17:04:47,692 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/WALs/b60eac9bda43,34957,1732122280382-splitting dir is empty, no logs to split. 2024-11-20T17:04:47,693 INFO [PEWorker-1 {}] master.SplitWALManager(105): b60eac9bda43,34957,1732122280382 WAL count=0, meta=false 2024-11-20T17:04:47,693 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if b60eac9bda43,34957,1732122280382 WAL splitting is done? wals=0, meta=false 2024-11-20T17:04:47,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN}] 2024-11-20T17:04:47,698 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:47,699 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:47,701 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:47,702 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:47,703 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:47,704 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:47,705 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:47,706 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:47,707 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:47,708 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:47,709 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:47,710 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:47,711 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:47,712 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:47,713 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:47,714 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T17:04:47,850 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(204): Hosts are {b60eac9bda43=0} racks are {/default-rack=0} 2024-11-20T17:04:47,850 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T17:04:47,850 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T17:04:47,850 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T17:04:47,850 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T17:04:47,850 INFO [b60eac9bda43:44751 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T17:04:47,850 INFO [b60eac9bda43:44751 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T17:04:47,850 DEBUG [b60eac9bda43:44751 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T17:04:47,851 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:47,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:47,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:47,852 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b60eac9bda43:34957 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:04:47,853 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, regionLocation=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:47,853 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=7b8c248055134523e6b12d02a033d557, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:47,853 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:47,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1, error=java.net.ConnectException: Call to address=b60eac9bda43:34957 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:47,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T17:04:47,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, regionLocation=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:47,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 from cache 2024-11-20T17:04:47,855 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:04:47,855 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,36999,1732122280556, seqNum=-1] 2024-11-20T17:04:47,855 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:04:47,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address b60eac9bda43:34957 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,856 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b60eac9bda43:34957 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:04:47,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,36999,1732122280556, seqNum=-1, error=java.net.ConnectException: Call to address=b60eac9bda43:34957 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,36999,1732122280556, seqNum=-1, error=java.net.ConnectException: Call to address=b60eac9bda43:34957 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,36999,1732122280556, seqNum=-1, error=java.net.ConnectException: Call to address=b60eac9bda43:34957 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,36999,1732122280556, seqNum=-1, error=java.net.ConnectException: Call to address=b60eac9bda43:34957 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,36999,1732122280556, seqNum=-1, error=java.net.ConnectException: Call to address=b60eac9bda43:34957 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,34957,1732122280382, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b60eac9bda43,36999,1732122280556, seqNum=-1, error=java.net.ConnectException: Call to address=b60eac9bda43:34957 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address b60eac9bda43:34957 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b60eac9bda43/172.17.0.2:34957 2024-11-20T17:04:47,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36535, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:04:47,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN because future has completed 2024-11-20T17:04:47,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:47,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN because future has completed 2024-11-20T17:04:47,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:47,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN because future has completed 2024-11-20T17:04:47,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN because future has completed 2024-11-20T17:04:47,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN because future has completed 2024-11-20T17:04:47,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:47,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN because future has completed 2024-11-20T17:04:47,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN because future has completed 2024-11-20T17:04:47,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:47,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN because future has completed 2024-11-20T17:04:47,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:47,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:47,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,36645,1732122280488}] 2024-11-20T17:04:47,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d48d4832d6d33017d258797576aabdc0, server=b60eac9bda43,36999,1732122280556}] 2024-11-20T17:04:48,017 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:48,018 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 052ad436a3fbddb02bb8de2342c3b9a7, NAME => 'test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-20T17:04:48,018 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,018 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:48,018 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,018 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,020 INFO [StoreOpener-052ad436a3fbddb02bb8de2342c3b9a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,021 INFO [StoreOpener-052ad436a3fbddb02bb8de2342c3b9a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 052ad436a3fbddb02bb8de2342c3b9a7 columnFamilyName family 2024-11-20T17:04:48,021 DEBUG [StoreOpener-052ad436a3fbddb02bb8de2342c3b9a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:48,022 INFO [StoreOpener-052ad436a3fbddb02bb8de2342c3b9a7-1 {}] regionserver.HStore(327): Store=052ad436a3fbddb02bb8de2342c3b9a7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:48,022 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,023 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,024 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,025 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,025 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,027 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,028 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 052ad436a3fbddb02bb8de2342c3b9a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61726476, jitterRate=-0.0802038311958313}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:48,028 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,028 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 052ad436a3fbddb02bb8de2342c3b9a7: Running coprocessor pre-open hook at 1732122288019Writing region info on filesystem at 1732122288019Initializing all the Stores at 1732122288020 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122288020Cleaning up temporary data from old regions at 1732122288025 (+5 ms)Running coprocessor post-open hooks at 1732122288028 (+3 ms)Region opened successfully at 1732122288028 2024-11-20T17:04:48,029 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7., pid=79, masterSystemTime=1732122288013 2024-11-20T17:04:48,031 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:48,032 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:48,032 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:48,032 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => d48d4832d6d33017d258797576aabdc0, NAME => 'test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-20T17:04:48,032 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,032 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:48,032 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,033 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,034 INFO [StoreOpener-d48d4832d6d33017d258797576aabdc0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,036 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-20T17:04:48,036 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,036 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,037 INFO [StoreOpener-d48d4832d6d33017d258797576aabdc0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d48d4832d6d33017d258797576aabdc0 columnFamilyName family 2024-11-20T17:04:48,037 DEBUG [StoreOpener-d48d4832d6d33017d258797576aabdc0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:48,038 INFO [StoreOpener-d48d4832d6d33017d258797576aabdc0-1 {}] regionserver.HStore(327): Store=d48d4832d6d33017d258797576aabdc0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:48,038 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,039 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,041 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,042 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,042 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,044 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,046 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened d48d4832d6d33017d258797576aabdc0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75330462, jitterRate=0.1225113570690155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:48,046 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,046 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for d48d4832d6d33017d258797576aabdc0: Running coprocessor pre-open hook at 1732122288033Writing region info on filesystem at 1732122288033Initializing all the Stores at 1732122288034 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122288034Cleaning up temporary data from old regions at 1732122288042 (+8 ms)Running coprocessor post-open hooks at 1732122288046 (+4 ms)Region opened successfully at 1732122288046 2024-11-20T17:04:48,047 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0., pid=82, masterSystemTime=1732122288013 2024-11-20T17:04:48,046 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-20T17:04:48,048 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,048 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d48d4832d6d33017d258797576aabdc0, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,048 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d48d4832d6d33017d258797576aabdc0, server=b60eac9bda43,36999,1732122280556 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,050 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:48,050 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:48,050 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:48,050 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b8c248055134523e6b12d02a033d557, NAME => 'test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-20T17:04:48,050 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,050 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:48,050 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,050 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,052 INFO [StoreOpener-7b8c248055134523e6b12d02a033d557-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,053 INFO [StoreOpener-7b8c248055134523e6b12d02a033d557-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b8c248055134523e6b12d02a033d557 columnFamilyName family 2024-11-20T17:04:48,053 DEBUG [StoreOpener-7b8c248055134523e6b12d02a033d557-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:48,053 INFO [StoreOpener-7b8c248055134523e6b12d02a033d557-1 {}] regionserver.HStore(327): Store=7b8c248055134523e6b12d02a033d557/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:48,054 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,054 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,056 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,056 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,056 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,058 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,059 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 7b8c248055134523e6b12d02a033d557; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63675024, jitterRate=-0.051168203353881836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:48,059 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,059 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 7b8c248055134523e6b12d02a033d557: Running coprocessor pre-open hook at 1732122288051Writing region info on filesystem at 1732122288051Initializing all the Stores at 1732122288051Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122288052 (+1 ms)Cleaning up temporary data from old regions at 1732122288056 (+4 ms)Running coprocessor post-open hooks at 1732122288059 (+3 ms)Region opened successfully at 1732122288059 2024-11-20T17:04:48,060 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557., pid=78, masterSystemTime=1732122288013 2024-11-20T17:04:48,063 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,063 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,063 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,063 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:48,063 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:48,063 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. 2024-11-20T17:04:48,063 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,063 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => a7a9afb4414c2ce282ecb8a226b732ef, NAME => 'test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-20T17:04:48,064 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,064 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:48,064 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,064 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,064 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,064 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,064 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,065 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,065 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:04:48,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,066 INFO [StoreOpener-a7a9afb4414c2ce282ecb8a226b732ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,067 INFO [StoreOpener-a7a9afb4414c2ce282ecb8a226b732ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7a9afb4414c2ce282ecb8a226b732ef columnFamilyName family 2024-11-20T17:04:48,067 DEBUG [StoreOpener-a7a9afb4414c2ce282ecb8a226b732ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:48,068 INFO [StoreOpener-a7a9afb4414c2ce282ecb8a226b732ef-1 {}] regionserver.HStore(327): Store=a7a9afb4414c2ce282ecb8a226b732ef/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:48,068 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,070 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,072 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,073 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,075 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,075 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,076 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,077 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened a7a9afb4414c2ce282ecb8a226b732ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59948565, jitterRate=-0.10669676959514618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:48,077 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7a9afb4414c2ce282ecb8a226b732ef 2024-11-20T17:04:48,077 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for a7a9afb4414c2ce282ecb8a226b732ef: Running coprocessor pre-open hook at 1732122288064Writing region info on filesystem at 1732122288064Initializing all the Stores at 1732122288065 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122288065Cleaning up temporary data from old regions at 1732122288073 (+8 ms)Running coprocessor post-open hooks at 1732122288077 (+4 ms)Region opened successfully at 1732122288077 2024-11-20T17:04:48,078 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef., pid=75, masterSystemTime=1732122288013 2024-11-20T17:04:48,079 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d48d4832d6d33017d258797576aabdc0, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,083 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,083 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,084 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,084 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,084 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,084 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,084 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,084 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,084 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d48d4832d6d33017d258797576aabdc0, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,085 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 19 msec 2024-11-20T17:04:48,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,087 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,087 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,105 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "b60eac9bda43" port: 36999 start_code: 1732122280556 } transition { transition_code: OPENED region_info { region_id: 1732122282892 table_name { namespace: "default" qualifier: "test" } start_key: "" end_key: "bbb" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 75 initiating_master_active_time: 1732122280659 }; retry (#0) after 1001ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-20T17:04:48,123 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. 2024-11-20T17:04:48,123 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => d01fc4378925287cb15b42165d6247f2, NAME => 'test,www,1732122282892.d01fc4378925287cb15b42165d6247f2.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-20T17:04:48,123 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,124 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,www,1732122282892.d01fc4378925287cb15b42165d6247f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:04:48,124 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,124 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,125 INFO [StoreOpener-d01fc4378925287cb15b42165d6247f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,126 INFO [StoreOpener-d01fc4378925287cb15b42165d6247f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d01fc4378925287cb15b42165d6247f2 columnFamilyName family 2024-11-20T17:04:48,126 DEBUG [StoreOpener-d01fc4378925287cb15b42165d6247f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:04:48,127 INFO [StoreOpener-d01fc4378925287cb15b42165d6247f2-1 {}] regionserver.HStore(327): Store=d01fc4378925287cb15b42165d6247f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:04:48,127 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,128 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,129 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-20T17:04:48,129 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,130 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,130 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,132 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,133 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened d01fc4378925287cb15b42165d6247f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63697874, jitterRate=-0.05082771182060242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T17:04:48,133 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d01fc4378925287cb15b42165d6247f2 2024-11-20T17:04:48,133 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for d01fc4378925287cb15b42165d6247f2: Running coprocessor pre-open hook at 1732122288124Writing region info on filesystem at 1732122288124Initializing all the Stores at 1732122288125 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732122288125Cleaning up temporary data from old regions at 1732122288130 (+5 ms)Running coprocessor post-open hooks at 1732122288133 (+3 ms)Region opened successfully at 1732122288133 2024-11-20T17:04:48,134 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732122282892.d01fc4378925287cb15b42165d6247f2., pid=76, masterSystemTime=1732122288118 2024-11-20T17:04:48,134 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,135 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,135 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "b60eac9bda43" port: 36645 start_code: 1732122280488 } transition { transition_code: OPENED region_info { region_id: 1732122282892 table_name { namespace: "default" qualifier: "test" } start_key: "www" end_key: "xxx" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1732122280659 }; retry (#0) after 1009ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-20T17:04:48,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,139 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,141 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,141 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318141 2024-11-20T17:04:48,142 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,142 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,143 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,144 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,144 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,36645,1732122280488 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,174 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348175 2024-11-20T17:04:48,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408175 2024-11-20T17:04:48,175 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-20T17:04:48,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,175 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,175 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,176 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,176 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,176 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,191 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,192 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,192 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,192 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,192 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,192 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,192 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,192 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,192 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b830d024dd19ff173c226582a1d0e05b, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-20T17:04:48,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,194 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,194 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,206 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,206 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,208 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318207 2024-11-20T17:04:48,208 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,208 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,211 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,226 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348226 2024-11-20T17:04:48,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408226 2024-11-20T17:04:48,226 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-20T17:04:48,226 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,226 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,227 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,227 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,227 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,227 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,235 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,240 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,242 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,243 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,243 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,243 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,243 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,243 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b4bd07e3663cc64a25dde8cd5845560, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,243 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-20T17:04:48,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,245 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,245 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,256 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,256 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,256 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,256 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,257 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318257 2024-11-20T17:04:48,257 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,257 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,259 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,260 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,260 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,260 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,273 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,273 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348273 2024-11-20T17:04:48,273 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408273 2024-11-20T17:04:48,273 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-20T17:04:48,273 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,273 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,273 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,273 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,273 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,273 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,273 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,274 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,274 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,281 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,288 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,288 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,288 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,288 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,288 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,288 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,288 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-20T17:04:48,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,290 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,290 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,297 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,297 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,297 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,298 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,298 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,298 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,299 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,299 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,299 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,299 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,299 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,299 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318299 2024-11-20T17:04:48,299 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,299 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,301 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,302 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,302 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,302 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,313 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348313 2024-11-20T17:04:48,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408313 2024-11-20T17:04:48,313 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-20T17:04:48,313 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,313 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,313 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,313 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,314 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,314 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,314 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,314 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,315 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,322 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,322 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,326 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,326 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,326 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,326 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,327 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,327 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,327 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b8c248055134523e6b12d02a033d557, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-20T17:04:48,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,329 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,329 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,339 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,339 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,339 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,340 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,340 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,340 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,340 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,340 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,340 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,340 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,340 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,340 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318340 2024-11-20T17:04:48,340 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,340 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,345 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,350 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,350 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348350 2024-11-20T17:04:48,350 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408350 2024-11-20T17:04:48,350 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T17:04:48,350 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,350 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,350 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,350 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,351 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,351 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,351 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,351 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,353 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,362 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,363 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,363 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,363 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e0f80ef67003cc41131beab8b406f39, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,364 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-20T17:04:48,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,365 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,365 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,373 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,374 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318374 2024-11-20T17:04:48,374 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,374 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,375 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,376 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,377 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,383 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,383 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348383 2024-11-20T17:04:48,383 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408383 2024-11-20T17:04:48,383 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T17:04:48,383 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,383 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,383 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,383 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,383 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,384 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,384 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,384 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,393 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,393 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,394 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,394 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,394 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,394 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,394 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,394 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,394 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,394 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,394 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d01fc4378925287cb15b42165d6247f2, server=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,394 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T17:04:48,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,395 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,395 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,406 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,407 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,407 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,407 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,407 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,407 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,407 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,407 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,407 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318407 2024-11-20T17:04:48,407 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,407 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,409 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,410 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,410 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,410 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,419 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,419 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348419 2024-11-20T17:04:48,419 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408419 2024-11-20T17:04:48,419 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-20T17:04:48,419 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,419 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,419 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,419 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,420 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,420 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,420 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,420 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,421 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,424 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,429 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,431 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,431 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9afb4414c2ce282ecb8a226b732ef, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,431 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T17:04:48,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,432 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,432 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,439 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,440 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,441 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318441 2024-11-20T17:04:48,441 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,441 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,442 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,443 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,443 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,444 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,450 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,450 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348450 2024-11-20T17:04:48,450 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408450 2024-11-20T17:04:48,450 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T17:04:48,450 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,450 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,450 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,450 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,451 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,451 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,451 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,451 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,452 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN 2024-11-20T17:04:48,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T17:04:48,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,458 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,458 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,463 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,465 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,466 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,466 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,466 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,466 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,466 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,466 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0e0f80ef67003cc41131beab8b406f39, ASSIGN to state=OFFLINE, location=null, table=test, region=0e0f80ef67003cc41131beab8b406f39 to restore RIT 2024-11-20T17:04:48,469 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318469 2024-11-20T17:04:48,469 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,469 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,470 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,471 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,471 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,471 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,478 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,479 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348478 2024-11-20T17:04:48,479 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408479 2024-11-20T17:04:48,479 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T17:04:48,479 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,479 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,479 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,479 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,479 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,479 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,479 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,479 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,480 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,482 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,485 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,485 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN 2024-11-20T17:04:48,485 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T17:04:48,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,486 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,486 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,498 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,498 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,498 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,499 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,499 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,499 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,499 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5b4bd07e3663cc64a25dde8cd5845560, ASSIGN to state=OFFLINE, location=null, table=test, region=5b4bd07e3663cc64a25dde8cd5845560 to restore RIT 2024-11-20T17:04:48,503 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318502 2024-11-20T17:04:48,503 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,503 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,505 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,507 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,507 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,507 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,512 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,512 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348512 2024-11-20T17:04:48,512 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408512 2024-11-20T17:04:48,512 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T17:04:48,512 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,512 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,512 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,512 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,512 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,513 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,513 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,516 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,516 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,517 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,518 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,518 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,518 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,518 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,518 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,518 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,518 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,518 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,518 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,518 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,518 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN 2024-11-20T17:04:48,518 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T17:04:48,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,519 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,519 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,539 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,539 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,539 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,540 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,540 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,540 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,540 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,540 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,540 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b830d024dd19ff173c226582a1d0e05b, ASSIGN to state=OFFLINE, location=null, table=test, region=b830d024dd19ff173c226582a1d0e05b to restore RIT 2024-11-20T17:04:48,540 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318540 2024-11-20T17:04:48,541 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,541 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,542 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,543 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,543 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,543 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,548 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,548 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348548 2024-11-20T17:04:48,548 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408548 2024-11-20T17:04:48,548 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T17:04:48,548 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,548 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,548 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,548 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,549 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,549 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,549 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,549 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,550 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN 2024-11-20T17:04:48,554 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T17:04:48,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,555 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,555 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,564 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,565 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,565 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,565 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,565 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b8c248055134523e6b12d02a033d557, ASSIGN to state=OFFLINE, location=null, table=test, region=7b8c248055134523e6b12d02a033d557 to restore RIT 2024-11-20T17:04:48,565 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318565 2024-11-20T17:04:48,565 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,565 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,566 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,569 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,569 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,569 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,574 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,574 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348574 2024-11-20T17:04:48,574 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408574 2024-11-20T17:04:48,574 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T17:04:48,575 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,575 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,575 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,575 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,575 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,576 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,576 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,580 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,581 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,581 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,581 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,581 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,581 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,581 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,581 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN 2024-11-20T17:04:48,581 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T17:04:48,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,582 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,582 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,590 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,590 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,590 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,590 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d48d4832d6d33017d258797576aabdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=d48d4832d6d33017d258797576aabdc0 to restore RIT 2024-11-20T17:04:48,590 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318590 2024-11-20T17:04:48,590 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,590 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,593 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,593 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,594 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,598 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348598 2024-11-20T17:04:48,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408598 2024-11-20T17:04:48,599 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-20T17:04:48,599 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,599 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,599 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,599 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,599 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,600 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,600 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,605 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,606 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN 2024-11-20T17:04:48,607 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T17:04:48,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,608 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,608 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,614 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,614 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,615 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=052ad436a3fbddb02bb8de2342c3b9a7, ASSIGN to state=OFFLINE, location=null, table=test, region=052ad436a3fbddb02bb8de2342c3b9a7 to restore RIT 2024-11-20T17:04:48,615 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318615 2024-11-20T17:04:48,615 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,615 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,616 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,617 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,617 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,617 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,622 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,622 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348622 2024-11-20T17:04:48,622 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408622 2024-11-20T17:04:48,622 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T17:04:48,622 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,622 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,622 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,622 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,623 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,623 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,623 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,627 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN 2024-11-20T17:04:48,628 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T17:04:48,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,629 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,629 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,662 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,662 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d01fc4378925287cb15b42165d6247f2, ASSIGN to state=OFFLINE, location=null, table=test, region=d01fc4378925287cb15b42165d6247f2 to restore RIT 2024-11-20T17:04:48,663 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318663 2024-11-20T17:04:48,663 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,663 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,664 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,665 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,671 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,671 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348671 2024-11-20T17:04:48,671 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408671 2024-11-20T17:04:48,671 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-20T17:04:48,671 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,672 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,672 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,672 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,672 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,672 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,673 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,674 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,674 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,674 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,674 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,674 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,674 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,677 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,677 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN 2024-11-20T17:04:48,677 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T17:04:48,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,678 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,679 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7a9afb4414c2ce282ecb8a226b732ef, ASSIGN to state=OFFLINE, location=null, table=test, region=a7a9afb4414c2ce282ecb8a226b732ef to restore RIT 2024-11-20T17:04:48,697 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318697 2024-11-20T17:04:48,697 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,697 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,698 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,699 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,700 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,704 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348704 2024-11-20T17:04:48,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408704 2024-11-20T17:04:48,704 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T17:04:48,704 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,704 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,704 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,705 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,705 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,705 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,705 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,706 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,709 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,709 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T17:04:48,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,710 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,710 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,722 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318722 2024-11-20T17:04:48,722 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,722 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,724 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,725 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,730 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,730 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348730 2024-11-20T17:04:48,730 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408730 2024-11-20T17:04:48,730 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T17:04:48,730 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,730 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,730 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,730 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,730 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,730 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,730 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,731 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,731 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,733 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,733 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,733 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,733 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,733 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,733 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,734 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,734 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,736 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.1732122282056, worker=b60eac9bda43,36645,1732122280488 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,736 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T17:04:48,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,737 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,737 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,749 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318749 2024-11-20T17:04:48,749 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,749 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,750 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,751 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,751 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,751 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,756 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,756 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348756 2024-11-20T17:04:48,756 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408756 2024-11-20T17:04:48,756 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T17:04:48,756 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,756 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,756 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,756 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,757 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,757 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,757 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,757 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,758 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T17:04:48,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,761 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T17:04:48,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,762 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,762 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,771 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44751 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318773 2024-11-20T17:04:48,774 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,774 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,775 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,776 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,776 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,776 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,781 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,781 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348781 2024-11-20T17:04:48,781 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408781 2024-11-20T17:04:48,781 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T17:04:48,781 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,781 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,781 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,781 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,781 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,781 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,781 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,782 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,782 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T17:04:48,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:04:48,785 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T17:04:48,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,786 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,786 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,798 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318798 2024-11-20T17:04:48,798 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,798 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,801 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,801 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,801 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,805 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,805 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348805 2024-11-20T17:04:48,805 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408805 2024-11-20T17:04:48,805 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T17:04:48,805 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,805 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,805 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,805 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,806 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,806 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,806 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,806 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,807 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T17:04:48,808 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,808 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,809 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,809 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,809 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,809 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T17:04:48,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,810 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,810 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,822 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318822 2024-11-20T17:04:48,822 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,822 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,823 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,824 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,824 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,824 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,828 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,828 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348828 2024-11-20T17:04:48,828 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408828 2024-11-20T17:04:48,828 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T17:04:48,829 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,829 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,829 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,829 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,829 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,829 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,829 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,829 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,830 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,831 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T17:04:48,831 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,832 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,832 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure b60eac9bda43%2C34957%2C1732122280382.meta.1732122282520.meta, worker=b60eac9bda43,36999,1732122280556 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T17:04:48,832 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T17:04:48,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,833 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,833 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,839 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,839 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,839 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,840 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318840 2024-11-20T17:04:48,840 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,840 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,841 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,842 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T17:04:48,842 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,843 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,846 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,846 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348846 2024-11-20T17:04:48,846 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408846 2024-11-20T17:04:48,846 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T17:04:48,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T17:04:48,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,847 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:48,847 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:48,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T17:04:48,847 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T17:04:48,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T17:04:48,847 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:04:48,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T17:04:48,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T17:04:48,849 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:04:48,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T17:04:48,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fd9711911c68abdd7f4a5f7d15d952e9, REOPEN/MOVE 2024-11-20T17:04:48,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true 2024-11-20T17:04:48,850 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T17:04:48,850 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T17:04:48,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=65 2024-11-20T17:04:48,851 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b60eac9bda43,36999,1732122280556, table=hbase:meta, region=1588230740 2024-11-20T17:04:48,851 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b60eac9bda43,36999,1732122280556, state=OPEN 2024-11-20T17:04:48,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:04:48,863 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,863 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,863 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:04:48,864 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732122318864 2024-11-20T17:04:48,864 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:04:48,864 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:04:48,865 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T17:04:48,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 052ad436a3fbddb02bb8de2342c3b9a7, server=b60eac9bda43,36999,1732122280556:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b60eac9bda43,34957,1732122280382, splitWal=true, meta=true exec-time=2.4090 sec 2024-11-20T17:04:48,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7a9afb4414c2ce282ecb8a226b732ef, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ae1354f6c79b7fe5dfb500ee1eb508d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b1a96a3d9afa0a1d03650e42aa3e2fd, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e0f5627de02ae3387d78a84cf60de116, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b4bd07e3663cc64a25dde8cd5845560, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12d417e0d4f0edac4d7bcadfbf305d96, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ead0f5940987b7964950b8f83e68ce7d, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d48d4832d6d33017d258797576aabdc0, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736a1f385c939ce1414e8e279f4e5834, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=be724bb9504bda880d6a757168537eb1, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5cc1a8141b65dd3101cc2eb478bf7fc, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e0f80ef67003cc41131beab8b406f39, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d9dc92a1bd5b9a54b65f86671af0015f, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=052ad436a3fbddb02bb8de2342c3b9a7, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13fb0e6117ef98d16b8be8bc4a741d1b, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad24e48b8f4e256db964da467a470ef8, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b830d024dd19ff173c226582a1d0e05b, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9de6e6c862796cabf8368e339eb39e67, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c22acb23a82c5ea894235ab5c0c16804, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b8c248055134523e6b12d02a033d557, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d0e6a5001fbe78b88c20912afb21151, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7bdef0be62b91bb7a8d2486eee55f68, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d01fc4378925287cb15b42165d6247f2, regionState=OPENING, lastHost=b60eac9bda43,34957,1732122280382, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17a1952b1af83907647c5d82cf786cc3, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=2 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd9711911c68abdd7f4a5f7d15d952e9, regionState=OPEN, lastHost=b60eac9bda43,36999,1732122280556, regionLocation=b60eac9bda43,36999,1732122280556, openSeqNum=5 2024-11-20T17:04:48,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c41b1abb64df3eda0e1232a964811d27, regionState=OPEN, lastHost=b60eac9bda43,36645,1732122280488, regionLocation=b60eac9bda43,36645,1732122280488, openSeqNum=2 2024-11-20T17:04:48,870 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T17:04:48,870 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732122348870 2024-11-20T17:04:48,870 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732122408870 2024-11-20T17:04:48,870 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T17:04:48,889 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 381), OpenFileDescriptor=622 (was 644), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=169 (was 175), ProcessCount=11 (was 11), AvailableMemoryMB=7522 (was 7578) 2024-11-20T17:04:48,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T17:04:48,890 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:04:48,890 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:04:48,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:48,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:48,890 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:04:48,891 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T17:04:48,891 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=189905736, stopped=false 2024-11-20T17:04:48,891 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b60eac9bda43,44751,1732122279653 2024-11-20T17:04:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:04:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:04:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:04:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:48,897 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:48,898 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:04:48,898 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:04:48,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:48,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:04:48,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:04:48,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:04:48,898 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b60eac9bda43,36645,1732122280488' ***** 2024-11-20T17:04:48,898 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T17:04:48,898 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b60eac9bda43,36999,1732122280556' ***** 2024-11-20T17:04:48,898 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T17:04:48,898 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:04:48,899 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:04:48,899 INFO [RS:2;b60eac9bda43:36999 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T17:04:48,899 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:04:48,899 INFO [RS:2;b60eac9bda43:36999 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T17:04:48,899 INFO [RS:1;b60eac9bda43:36645 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T17:04:48,899 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:04:48,899 INFO [RS:1;b60eac9bda43:36645 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T17:04:48,899 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for ad24e48b8f4e256db964da467a470ef8 2024-11-20T17:04:48,899 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 12d417e0d4f0edac4d7bcadfbf305d96 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for c22acb23a82c5ea894235ab5c0c16804 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for c7bdef0be62b91bb7a8d2486eee55f68 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for be724bb9504bda880d6a757168537eb1 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ae1354f6c79b7fe5dfb500ee1eb508d 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for c41b1abb64df3eda0e1232a964811d27 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for d9dc92a1bd5b9a54b65f86671af0015f 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for e0f5627de02ae3387d78a84cf60de116 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(3091): Received CLOSE for ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(959): stopping server b60eac9bda43,36645,1732122280488 2024-11-20T17:04:48,900 INFO [RS:1;b60eac9bda43:36645 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:04:48,900 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad24e48b8f4e256db964da467a470ef8, disabling compactions & flushes 2024-11-20T17:04:48,901 INFO [RS:1;b60eac9bda43:36645 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b60eac9bda43:36645. 2024-11-20T17:04:48,901 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:48,901 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:48,901 DEBUG [RS:1;b60eac9bda43:36645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 052ad436a3fbddb02bb8de2342c3b9a7 2024-11-20T17:04:48,901 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. after waiting 0 ms 2024-11-20T17:04:48,901 DEBUG [RS:1;b60eac9bda43:36645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:48,901 INFO [regionserver/b60eac9bda43:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b8c248055134523e6b12d02a033d557 2024-11-20T17:04:48,901 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:48,901 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12d417e0d4f0edac4d7bcadfbf305d96, disabling compactions & flushes 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 9de6e6c862796cabf8368e339eb39e67 2024-11-20T17:04:48,901 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 17a1952b1af83907647c5d82cf786cc3 2024-11-20T17:04:48,901 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for f5cc1a8141b65dd3101cc2eb478bf7fc 2024-11-20T17:04:48,901 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-20T17:04:48,901 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. after waiting 0 ms 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:48,901 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 9b1a96a3d9afa0a1d03650e42aa3e2fd 2024-11-20T17:04:48,901 DEBUG [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1325): Online Regions={ad24e48b8f4e256db964da467a470ef8=test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8., c22acb23a82c5ea894235ab5c0c16804=test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804., c7bdef0be62b91bb7a8d2486eee55f68=test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68., be724bb9504bda880d6a757168537eb1=test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1., 0ae1354f6c79b7fe5dfb500ee1eb508d=test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d., c41b1abb64df3eda0e1232a964811d27=test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27., d9dc92a1bd5b9a54b65f86671af0015f=test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f., e0f5627de02ae3387d78a84cf60de116=test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116., ead0f5940987b7964950b8f83e68ce7d=test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d.} 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 7d0e6a5001fbe78b88c20912afb21151 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 13fb0e6117ef98d16b8be8bc4a741d1b 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for d48d4832d6d33017d258797576aabdc0 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 736a1f385c939ce1414e8e279f4e5834 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(959): stopping server b60eac9bda43,36999,1732122280556 2024-11-20T17:04:48,901 DEBUG [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1351): Waiting on 0ae1354f6c79b7fe5dfb500ee1eb508d, ad24e48b8f4e256db964da467a470ef8, be724bb9504bda880d6a757168537eb1, c22acb23a82c5ea894235ab5c0c16804, c41b1abb64df3eda0e1232a964811d27, c7bdef0be62b91bb7a8d2486eee55f68, d01fc4378925287cb15b42165d6247f2, d9dc92a1bd5b9a54b65f86671af0015f, e0f5627de02ae3387d78a84cf60de116, ead0f5940987b7964950b8f83e68ce7d 2024-11-20T17:04:48,901 INFO [RS:2;b60eac9bda43:36999 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:04:48,902 INFO [RS:2;b60eac9bda43:36999 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b60eac9bda43:36999. 2024-11-20T17:04:48,902 DEBUG [RS:2;b60eac9bda43:36999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:04:48,902 DEBUG [RS:2;b60eac9bda43:36999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:48,902 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T17:04:48,902 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T17:04:48,902 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T17:04:48,902 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T17:04:48,903 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-20T17:04:48,903 DEBUG [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(1351): Waiting on 052ad436a3fbddb02bb8de2342c3b9a7, 12d417e0d4f0edac4d7bcadfbf305d96, 13fb0e6117ef98d16b8be8bc4a741d1b, 1588230740, 17a1952b1af83907647c5d82cf786cc3, 736a1f385c939ce1414e8e279f4e5834, 7b8c248055134523e6b12d02a033d557, 7d0e6a5001fbe78b88c20912afb21151, 9b1a96a3d9afa0a1d03650e42aa3e2fd, 9de6e6c862796cabf8368e339eb39e67, a7a9afb4414c2ce282ecb8a226b732ef, d48d4832d6d33017d258797576aabdc0, f5cc1a8141b65dd3101cc2eb478bf7fc, fd9711911c68abdd7f4a5f7d15d952e9 2024-11-20T17:04:48,903 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:04:48,903 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:04:48,903 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:04:48,903 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:04:48,903 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:04:48,903 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-20T17:04:48,905 INFO [regionserver/b60eac9bda43:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:04:48,907 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/12d417e0d4f0edac4d7bcadfbf305d96/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,908 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/ad24e48b8f4e256db964da467a470ef8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,908 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:48,908 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12d417e0d4f0edac4d7bcadfbf305d96: Waiting for close lock at 1732122288901Running coprocessor pre-close hooks at 1732122288901Disabling compacts and flushes for region at 1732122288901Disabling writes for close at 1732122288901Writing region close event to WAL at 1732122288903 (+2 ms)Running coprocessor post-close hooks at 1732122288908 (+5 ms)Closed at 1732122288908 2024-11-20T17:04:48,908 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732122282892.12d417e0d4f0edac4d7bcadfbf305d96. 2024-11-20T17:04:48,908 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 052ad436a3fbddb02bb8de2342c3b9a7, disabling compactions & flushes 2024-11-20T17:04:48,908 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:48,908 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:48,908 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. after waiting 0 ms 2024-11-20T17:04:48,909 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:48,909 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:48,909 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad24e48b8f4e256db964da467a470ef8: Waiting for close lock at 1732122288900Running coprocessor pre-close hooks at 1732122288900Disabling compacts and flushes for region at 1732122288900Disabling writes for close at 1732122288901 (+1 ms)Writing region close event to WAL at 1732122288902 (+1 ms)Running coprocessor post-close hooks at 1732122288908 (+6 ms)Closed at 1732122288909 (+1 ms) 2024-11-20T17:04:48,909 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732122282892.ad24e48b8f4e256db964da467a470ef8. 2024-11-20T17:04:48,909 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c22acb23a82c5ea894235ab5c0c16804, disabling compactions & flushes 2024-11-20T17:04:48,909 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:48,909 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:48,909 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. after waiting 0 ms 2024-11-20T17:04:48,909 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:48,913 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/052ad436a3fbddb02bb8de2342c3b9a7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,914 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c22acb23a82c5ea894235ab5c0c16804/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,914 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:48,914 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:48,914 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 052ad436a3fbddb02bb8de2342c3b9a7: Waiting for close lock at 1732122288908Running coprocessor pre-close hooks at 1732122288908Disabling compacts and flushes for region at 1732122288908Disabling writes for close at 1732122288909 (+1 ms)Writing region close event to WAL at 1732122288909Running coprocessor post-close hooks at 1732122288914 (+5 ms)Closed at 1732122288914 2024-11-20T17:04:48,914 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c22acb23a82c5ea894235ab5c0c16804: Waiting for close lock at 1732122288909Running coprocessor pre-close hooks at 1732122288909Disabling compacts and flushes for region at 1732122288909Disabling writes for close at 1732122288909Writing region close event to WAL at 1732122288910 (+1 ms)Running coprocessor post-close hooks at 1732122288914 (+4 ms)Closed at 1732122288914 2024-11-20T17:04:48,914 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732122282892.052ad436a3fbddb02bb8de2342c3b9a7. 2024-11-20T17:04:48,914 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732122282892.c22acb23a82c5ea894235ab5c0c16804. 2024-11-20T17:04:48,915 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b8c248055134523e6b12d02a033d557, disabling compactions & flushes 2024-11-20T17:04:48,915 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c7bdef0be62b91bb7a8d2486eee55f68, disabling compactions & flushes 2024-11-20T17:04:48,915 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:48,915 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:48,915 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:48,915 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:48,915 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. after waiting 0 ms 2024-11-20T17:04:48,915 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. after waiting 0 ms 2024-11-20T17:04:48,915 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:48,915 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:48,916 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/info/8dc780d4e26d4e01a37a6cacc4ec9548 is 138, key is test,eee,1732122282892.5b4bd07e3663cc64a25dde8cd5845560./info:regioninfo/1732122287852/Put/seqid=0 2024-11-20T17:04:48,920 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7b8c248055134523e6b12d02a033d557/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,920 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c7bdef0be62b91bb7a8d2486eee55f68/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,920 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:48,920 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b8c248055134523e6b12d02a033d557: Waiting for close lock at 1732122288915Running coprocessor pre-close hooks at 1732122288915Disabling compacts and flushes for region at 1732122288915Disabling writes for close at 1732122288915Writing region close event to WAL at 1732122288915Running coprocessor post-close hooks at 1732122288920 (+5 ms)Closed at 1732122288920 2024-11-20T17:04:48,920 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:48,920 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c7bdef0be62b91bb7a8d2486eee55f68: Waiting for close lock at 1732122288915Running coprocessor pre-close hooks at 1732122288915Disabling compacts and flushes for region at 1732122288915Disabling writes for close at 1732122288915Writing region close event to WAL at 1732122288915Running coprocessor post-close hooks at 1732122288920 (+5 ms)Closed at 1732122288920 2024-11-20T17:04:48,920 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732122282892.7b8c248055134523e6b12d02a033d557. 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732122282892.c7bdef0be62b91bb7a8d2486eee55f68. 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9de6e6c862796cabf8368e339eb39e67, disabling compactions & flushes 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing be724bb9504bda880d6a757168537eb1, disabling compactions & flushes 2024-11-20T17:04:48,921 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:48,921 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. after waiting 0 ms 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. after waiting 0 ms 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:48,921 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:48,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741869_1045 (size=8177) 2024-11-20T17:04:48,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741869_1045 (size=8177) 2024-11-20T17:04:48,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741869_1045 (size=8177) 2024-11-20T17:04:48,930 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/info/8dc780d4e26d4e01a37a6cacc4ec9548 2024-11-20T17:04:48,931 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/be724bb9504bda880d6a757168537eb1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,932 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:48,932 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for be724bb9504bda880d6a757168537eb1: Waiting for close lock at 1732122288921Running coprocessor pre-close hooks at 1732122288921Disabling compacts and flushes for region at 1732122288921Disabling writes for close at 1732122288921Writing region close event to WAL at 1732122288921Running coprocessor post-close hooks at 1732122288932 (+11 ms)Closed at 1732122288932 2024-11-20T17:04:48,932 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732122282892.be724bb9504bda880d6a757168537eb1. 2024-11-20T17:04:48,932 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ae1354f6c79b7fe5dfb500ee1eb508d, disabling compactions & flushes 2024-11-20T17:04:48,932 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:48,932 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:48,932 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. after waiting 0 ms 2024-11-20T17:04:48,932 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:48,935 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/9de6e6c862796cabf8368e339eb39e67/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,936 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:48,936 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9de6e6c862796cabf8368e339eb39e67: Waiting for close lock at 1732122288921Running coprocessor pre-close hooks at 1732122288921Disabling compacts and flushes for region at 1732122288921Disabling writes for close at 1732122288921Writing region close event to WAL at 1732122288921Running coprocessor post-close hooks at 1732122288936 (+15 ms)Closed at 1732122288936 2024-11-20T17:04:48,936 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732122282892.9de6e6c862796cabf8368e339eb39e67. 2024-11-20T17:04:48,936 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17a1952b1af83907647c5d82cf786cc3, disabling compactions & flushes 2024-11-20T17:04:48,936 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:48,936 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:48,936 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. after waiting 0 ms 2024-11-20T17:04:48,936 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:48,941 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/0ae1354f6c79b7fe5dfb500ee1eb508d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,941 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/.tmp/info/8dc780d4e26d4e01a37a6cacc4ec9548 as hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/info/8dc780d4e26d4e01a37a6cacc4ec9548 2024-11-20T17:04:48,941 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/17a1952b1af83907647c5d82cf786cc3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,941 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:48,941 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ae1354f6c79b7fe5dfb500ee1eb508d: Waiting for close lock at 1732122288932Running coprocessor pre-close hooks at 1732122288932Disabling compacts and flushes for region at 1732122288932Disabling writes for close at 1732122288932Writing region close event to WAL at 1732122288936 (+4 ms)Running coprocessor post-close hooks at 1732122288941 (+5 ms)Closed at 1732122288941 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732122282892.0ae1354f6c79b7fe5dfb500ee1eb508d. 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c41b1abb64df3eda0e1232a964811d27, disabling compactions & flushes 2024-11-20T17:04:48,942 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. after waiting 0 ms 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:48,942 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17a1952b1af83907647c5d82cf786cc3: Waiting for close lock at 1732122288936Running coprocessor pre-close hooks at 1732122288936Disabling compacts and flushes for region at 1732122288936Disabling writes for close at 1732122288936Writing region close event to WAL at 1732122288937 (+1 ms)Running coprocessor post-close hooks at 1732122288942 (+5 ms)Closed at 1732122288942 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732122282892.17a1952b1af83907647c5d82cf786cc3. 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f5cc1a8141b65dd3101cc2eb478bf7fc, disabling compactions & flushes 2024-11-20T17:04:48,942 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:48,942 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:48,943 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. after waiting 0 ms 2024-11-20T17:04:48,943 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:48,947 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/c41b1abb64df3eda0e1232a964811d27/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,948 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:48,948 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c41b1abb64df3eda0e1232a964811d27: Waiting for close lock at 1732122288942Running coprocessor pre-close hooks at 1732122288942Disabling compacts and flushes for region at 1732122288942Disabling writes for close at 1732122288942Writing region close event to WAL at 1732122288942Running coprocessor post-close hooks at 1732122288948 (+6 ms)Closed at 1732122288948 2024-11-20T17:04:48,949 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732122282892.c41b1abb64df3eda0e1232a964811d27. 2024-11-20T17:04:48,949 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d9dc92a1bd5b9a54b65f86671af0015f, disabling compactions & flushes 2024-11-20T17:04:48,949 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:48,949 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:48,949 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. after waiting 0 ms 2024-11-20T17:04:48,949 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:48,950 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/f5cc1a8141b65dd3101cc2eb478bf7fc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,951 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:48,951 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f5cc1a8141b65dd3101cc2eb478bf7fc: Waiting for close lock at 1732122288942Running coprocessor pre-close hooks at 1732122288942Disabling compacts and flushes for region at 1732122288942Disabling writes for close at 1732122288943 (+1 ms)Writing region close event to WAL at 1732122288943Running coprocessor post-close hooks at 1732122288951 (+8 ms)Closed at 1732122288951 2024-11-20T17:04:48,951 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732122282892.f5cc1a8141b65dd3101cc2eb478bf7fc. 2024-11-20T17:04:48,951 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd9711911c68abdd7f4a5f7d15d952e9, disabling compactions & flushes 2024-11-20T17:04:48,951 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:48,951 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:48,952 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. after waiting 0 ms 2024-11-20T17:04:48,952 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:48,952 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/info/8dc780d4e26d4e01a37a6cacc4ec9548, entries=24, sequenceid=75, filesize=8.0 K 2024-11-20T17:04:48,954 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 51ms, sequenceid=75, compaction requested=false 2024-11-20T17:04:48,955 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d9dc92a1bd5b9a54b65f86671af0015f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,955 INFO [regionserver/b60eac9bda43:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T17:04:48,956 INFO [regionserver/b60eac9bda43:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T17:04:48,956 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:48,956 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d9dc92a1bd5b9a54b65f86671af0015f: Waiting for close lock at 1732122288949Running coprocessor pre-close hooks at 1732122288949Disabling compacts and flushes for region at 1732122288949Disabling writes for close at 1732122288949Writing region close event to WAL at 1732122288950 (+1 ms)Running coprocessor post-close hooks at 1732122288956 (+6 ms)Closed at 1732122288956 2024-11-20T17:04:48,956 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732122282892.d9dc92a1bd5b9a54b65f86671af0015f. 2024-11-20T17:04:48,956 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e0f5627de02ae3387d78a84cf60de116, disabling compactions & flushes 2024-11-20T17:04:48,956 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:48,956 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:48,956 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. after waiting 0 ms 2024-11-20T17:04:48,956 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:48,959 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/fd9711911c68abdd7f4a5f7d15d952e9/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-20T17:04:48,960 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:48,960 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd9711911c68abdd7f4a5f7d15d952e9: Waiting for close lock at 1732122288951Running coprocessor pre-close hooks at 1732122288951Disabling compacts and flushes for region at 1732122288951Disabling writes for close at 1732122288952 (+1 ms)Writing region close event to WAL at 1732122288953 (+1 ms)Running coprocessor post-close hooks at 1732122288960 (+7 ms)Closed at 1732122288960 2024-11-20T17:04:48,960 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732122282892.fd9711911c68abdd7f4a5f7d15d952e9. 2024-11-20T17:04:48,960 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9b1a96a3d9afa0a1d03650e42aa3e2fd, disabling compactions & flushes 2024-11-20T17:04:48,960 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:48,960 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-20T17:04:48,960 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:48,961 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. after waiting 0 ms 2024-11-20T17:04:48,961 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:48,961 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T17:04:48,961 INFO [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:04:48,961 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732122288903Running coprocessor pre-close hooks at 1732122288903Disabling compacts and flushes for region at 1732122288903Disabling writes for close at 1732122288903Obtaining lock to block concurrent updates at 1732122288903Preparing flush snapshotting stores in 1588230740 at 1732122288903Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1732122288909 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732122288911 (+2 ms)Flushing 1588230740/info: creating writer at 1732122288911Flushing 1588230740/info: appending metadata at 1732122288915 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732122288915Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e4b358: reopening flushed file at 1732122288940 (+25 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 51ms, sequenceid=75, compaction requested=false at 1732122288954 (+14 ms)Writing region close event to WAL at 1732122288956 (+2 ms)Running coprocessor post-close hooks at 1732122288961 (+5 ms)Closed at 1732122288961 2024-11-20T17:04:48,961 DEBUG [RS_CLOSE_META-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T17:04:48,962 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/e0f5627de02ae3387d78a84cf60de116/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,963 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:48,963 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e0f5627de02ae3387d78a84cf60de116: Waiting for close lock at 1732122288956Running coprocessor pre-close hooks at 1732122288956Disabling compacts and flushes for region at 1732122288956Disabling writes for close at 1732122288956Writing region close event to WAL at 1732122288957 (+1 ms)Running coprocessor post-close hooks at 1732122288963 (+6 ms)Closed at 1732122288963 2024-11-20T17:04:48,963 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732122282892.e0f5627de02ae3387d78a84cf60de116. 2024-11-20T17:04:48,963 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ead0f5940987b7964950b8f83e68ce7d, disabling compactions & flushes 2024-11-20T17:04:48,963 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:48,963 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:48,963 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. after waiting 0 ms 2024-11-20T17:04:48,963 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:48,965 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/9b1a96a3d9afa0a1d03650e42aa3e2fd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,966 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:48,966 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9b1a96a3d9afa0a1d03650e42aa3e2fd: Waiting for close lock at 1732122288960Running coprocessor pre-close hooks at 1732122288960Disabling compacts and flushes for region at 1732122288960Disabling writes for close at 1732122288961 (+1 ms)Writing region close event to WAL at 1732122288961Running coprocessor post-close hooks at 1732122288966 (+5 ms)Closed at 1732122288966 2024-11-20T17:04:48,966 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732122282892.9b1a96a3d9afa0a1d03650e42aa3e2fd. 2024-11-20T17:04:48,966 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7d0e6a5001fbe78b88c20912afb21151, disabling compactions & flushes 2024-11-20T17:04:48,967 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:48,967 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:48,967 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. after waiting 0 ms 2024-11-20T17:04:48,967 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:48,968 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/ead0f5940987b7964950b8f83e68ce7d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,968 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:48,968 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ead0f5940987b7964950b8f83e68ce7d: Waiting for close lock at 1732122288963Running coprocessor pre-close hooks at 1732122288963Disabling compacts and flushes for region at 1732122288963Disabling writes for close at 1732122288963Writing region close event to WAL at 1732122288964 (+1 ms)Running coprocessor post-close hooks at 1732122288968 (+4 ms)Closed at 1732122288968 2024-11-20T17:04:48,968 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732122282892.ead0f5940987b7964950b8f83e68ce7d. 2024-11-20T17:04:48,971 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/7d0e6a5001fbe78b88c20912afb21151/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,972 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:48,972 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7d0e6a5001fbe78b88c20912afb21151: Waiting for close lock at 1732122288966Running coprocessor pre-close hooks at 1732122288966Disabling compacts and flushes for region at 1732122288966Disabling writes for close at 1732122288967 (+1 ms)Writing region close event to WAL at 1732122288967Running coprocessor post-close hooks at 1732122288972 (+5 ms)Closed at 1732122288972 2024-11-20T17:04:48,972 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732122282892.7d0e6a5001fbe78b88c20912afb21151. 2024-11-20T17:04:48,972 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 13fb0e6117ef98d16b8be8bc4a741d1b, disabling compactions & flushes 2024-11-20T17:04:48,972 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:48,972 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:48,972 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. after waiting 0 ms 2024-11-20T17:04:48,972 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:48,977 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/13fb0e6117ef98d16b8be8bc4a741d1b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,978 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:48,978 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 13fb0e6117ef98d16b8be8bc4a741d1b: Waiting for close lock at 1732122288972Running coprocessor pre-close hooks at 1732122288972Disabling compacts and flushes for region at 1732122288972Disabling writes for close at 1732122288972Writing region close event to WAL at 1732122288973 (+1 ms)Running coprocessor post-close hooks at 1732122288977 (+4 ms)Closed at 1732122288977 2024-11-20T17:04:48,978 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732122282892.13fb0e6117ef98d16b8be8bc4a741d1b. 2024-11-20T17:04:48,978 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d48d4832d6d33017d258797576aabdc0, disabling compactions & flushes 2024-11-20T17:04:48,978 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:48,978 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:48,978 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. after waiting 0 ms 2024-11-20T17:04:48,978 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:48,983 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/d48d4832d6d33017d258797576aabdc0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,983 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:48,984 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d48d4832d6d33017d258797576aabdc0: Waiting for close lock at 1732122288978Running coprocessor pre-close hooks at 1732122288978Disabling compacts and flushes for region at 1732122288978Disabling writes for close at 1732122288978Writing region close event to WAL at 1732122288979 (+1 ms)Running coprocessor post-close hooks at 1732122288983 (+4 ms)Closed at 1732122288983 2024-11-20T17:04:48,984 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732122282892.d48d4832d6d33017d258797576aabdc0. 2024-11-20T17:04:48,984 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 736a1f385c939ce1414e8e279f4e5834, disabling compactions & flushes 2024-11-20T17:04:48,984 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:48,984 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:48,984 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. after waiting 0 ms 2024-11-20T17:04:48,984 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:48,989 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/data/default/test/736a1f385c939ce1414e8e279f4e5834/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:04:48,989 INFO [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:48,989 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 736a1f385c939ce1414e8e279f4e5834: Waiting for close lock at 1732122288984Running coprocessor pre-close hooks at 1732122288984Disabling compacts and flushes for region at 1732122288984Disabling writes for close at 1732122288984Writing region close event to WAL at 1732122288985 (+1 ms)Running coprocessor post-close hooks at 1732122288989 (+4 ms)Closed at 1732122288989 2024-11-20T17:04:48,989 DEBUG [RS_CLOSE_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732122282892.736a1f385c939ce1414e8e279f4e5834. 2024-11-20T17:04:49,102 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(976): stopping server b60eac9bda43,36645,1732122280488; all regions closed. 2024-11-20T17:04:49,103 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.HRegionServer(976): stopping server b60eac9bda43,36999,1732122280556; all regions closed. 2024-11-20T17:04:49,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741833_1009 (size=5194) 2024-11-20T17:04:49,107 WARN [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef., aborting... java.io.IOException: Failed to report opened region to master: test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:04:49,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741833_1009 (size=5194) 2024-11-20T17:04:49,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741833_1009 (size=5194) 2024-11-20T17:04:49,109 ERROR [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2440): ***** ABORTING region server b60eac9bda43,36999,1732122280556: Failed to open region test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,,1732122282892.a7a9afb4414c2ce282ecb8a226b732ef. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:04:49,109 ERROR [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T17:04:49,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741865_1041 (size=4192) 2024-11-20T17:04:49,109 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T17:04:49,111 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T17:04:49,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741865_1041 (size=4192) 2024-11-20T17:04:49,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741865_1041 (size=4192) 2024-11-20T17:04:49,114 DEBUG [RS:1;b60eac9bda43:36645 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs 2024-11-20T17:04:49,114 INFO [RS:1;b60eac9bda43:36645 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b60eac9bda43%2C36645%2C1732122280488:(num 1732122282057) 2024-11-20T17:04:49,114 DEBUG [RS:1;b60eac9bda43:36645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:49,114 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:04:49,114 INFO [RS:1;b60eac9bda43:36645 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:04:49,114 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T17:04:49,115 INFO [RS:1;b60eac9bda43:36645 {}] hbase.ChoreService(370): Chore service for: regionserver/b60eac9bda43:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T17:04:49,115 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T17:04:49,115 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T17:04:49,115 INFO [regionserver/b60eac9bda43:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:04:49,115 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T17:04:49,115 INFO [RS:1;b60eac9bda43:36645 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:04:49,115 WARN [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:04:49,116 WARN [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,www,1732122282892.d01fc4378925287cb15b42165d6247f2., aborting... java.io.IOException: Failed to report opened region to master: test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:04:49,116 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T17:04:49,116 ERROR [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2440): ***** ABORTING region server b60eac9bda43,36645,1732122280488: Failed to open region test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,www,1732122282892.d01fc4378925287cb15b42165d6247f2. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:04:49,116 DEBUG [RS:2;b60eac9bda43:36999 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs 2024-11-20T17:04:49,116 ERROR [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T17:04:49,116 INFO [RS:2;b60eac9bda43:36999 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b60eac9bda43%2C36999%2C1732122280556.meta:.meta(num 1732122287110) 2024-11-20T17:04:49,116 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T17:04:49,117 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T17:04:49,117 INFO [RS:1;b60eac9bda43:36645 {}] executor.ExecutorService(108): Executor-10-RS_OPEN_REGION-regionserver/b60eac9bda43:0 had [Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #39 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #40 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-20T17:04:49,117 INFO [RS:1;b60eac9bda43:36645 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36645 2024-11-20T17:04:49,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741834_1010 (size=6800) 2024-11-20T17:04:49,119 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T17:04:49,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741834_1010 (size=6800) 2024-11-20T17:04:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741834_1010 (size=6800) 2024-11-20T17:04:49,121 DEBUG [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T17:04:49,123 DEBUG [RS:2;b60eac9bda43:36999 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/oldWALs 2024-11-20T17:04:49,123 INFO [RS:2;b60eac9bda43:36999 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b60eac9bda43%2C36999%2C1732122280556:(num 1732122282056) 2024-11-20T17:04:49,123 DEBUG [RS:2;b60eac9bda43:36999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:04:49,123 INFO [RS:2;b60eac9bda43:36999 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:04:49,123 INFO [RS:2;b60eac9bda43:36999 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:04:49,123 INFO [RS:2;b60eac9bda43:36999 {}] hbase.ChoreService(370): Chore service for: regionserver/b60eac9bda43:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T17:04:49,124 INFO [RS:2;b60eac9bda43:36999 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:04:49,124 INFO [regionserver/b60eac9bda43:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:04:49,124 INFO [RS:2;b60eac9bda43:36999 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36999 2024-11-20T17:04:49,128 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 504342048 }, "NonHeapMemoryUsage": { "committed": 164233216, "init": 7667712, "max": -1, "used": 161574408 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39092, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 115, "ProcessCallTime_min": 0, "ProcessCallTime_max": 19, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 9, "ProcessCallTime_75th_percentile": 14, "ProcessCallTime_90th_percentile": 17, "ProcessCallTime_95th_percentile": 18, "ProcessCallTime_98th_percentile": 18, "ProcessCallTime_99th_percentile": 18, "ProcessCallTime_99.9th_percentile": 18, "ProcessCallTime_TimeRangeCount_0-1": 115, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 115, "QueueCallTime_min": 0, "QueueCallTime_max": 28, "QueueCallTime_mean": 2, "QueueCallTime_25th_percentile": 7, "QueueCallTime_median": 14, "QueueCallTime_75th_percentile": 21, "QueueCallTime_90th_percentile": 25, "QueueCallTime_95th_percentile": 26, "QueueCallTime_98th_percentile": 27, "QueueCallTime_99th_percentile": 27, "QueueCallTime_99.9th_percentile": 27, "QueueCallTime_TimeRangeCount_0-1": 115, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 115, "TotalCallTime_min": 0, "TotalCallTime_max": 30, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 15, "TotalCallTime_75th_percentile": 22, "TotalCallTime_90th_percentile": 27, "TotalCallTime_95th_percentile": 28, "TotalCallTime_98th_percentile": 29, "TotalCallTime_99th_percentile": 29, "TotalCallTime_99.9th_percentile": 29, "TotalCallTime_TimeRangeCount_0-1": 115, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 115, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 594, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 115, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 115, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 307, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 115, "sentBytes": 54860 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57068", "tag.serverName": "b60eac9bda43,36999,1732122280556", "tag.clusterId": "a3098b73-f92a-4174-8ea2-e2512d71554a", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43", "regionCount": 13, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732122280556, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.6, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7456, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 16, "totalRowActionRequestCount": 8, "readRequestCount": 8, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 1, "Put_max": 1, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 1, "Put_75th_percentile": 1, "Put_90th_percentile": 1, "Put_95th_percentile": 1, "Put_98th_percentile": 1, "Put_99th_percentile": 1, "Put_99.9th_percentile": 1, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46694, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46694, "FlushOutputSize_max": 46694, "FlushOutputSize_mean": 46694, "FlushOutputSize_25th_percentile": 46694, "FlushOutputSize_median": 46694, "FlushOutputSize_75th_percentile": 46694, "FlushOutputSize_90th_percentile": 46694, "FlushOutputSize_95th_percentile": 46694, "FlushOutputSize_98th_percentile": 46694, "FlushOutputSize_99th_percentile": 46694, "FlushOutputSize_99.9th_percentile": 46694, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 245, "FlushTime_max": 245, "FlushTime_mean": 245, "FlushTime_25th_percentile": 245, "FlushTime_median": 245, "FlushTime_75th_percentile": 245, "FlushTime_90th_percentile": 245, "FlushTime_95th_percentile": 245, "FlushTime_98th_percentile": 245, "FlushTime_99th_percentile": 245, "FlushTime_99.9th_percentile": 245, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T17:04:49,131 INFO [RS_OPEN_REGION-regionserver/b60eac9bda43:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 504342048 }, "NonHeapMemoryUsage": { "committed": 164298752, "init": 7667712, "max": -1, "used": 161647464 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39092, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 115, "ProcessCallTime_min": 0, "ProcessCallTime_max": 19, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 9, "ProcessCallTime_75th_percentile": 14, "ProcessCallTime_90th_percentile": 17, "ProcessCallTime_95th_percentile": 18, "ProcessCallTime_98th_percentile": 18, "ProcessCallTime_99th_percentile": 18, "ProcessCallTime_99.9th_percentile": 18, "ProcessCallTime_TimeRangeCount_0-1": 115, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 115, "QueueCallTime_min": 0, "QueueCallTime_max": 28, "QueueCallTime_mean": 2, "QueueCallTime_25th_percentile": 7, "QueueCallTime_median": 14, "QueueCallTime_75th_percentile": 21, "QueueCallTime_90th_percentile": 25, "QueueCallTime_95th_percentile": 26, "QueueCallTime_98th_percentile": 27, "QueueCallTime_99th_percentile": 27, "QueueCallTime_99.9th_percentile": 27, "QueueCallTime_TimeRangeCount_0-1": 115, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 115, "TotalCallTime_min": 0, "TotalCallTime_max": 30, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 15, "TotalCallTime_75th_percentile": 22, "TotalCallTime_90th_percentile": 27, "TotalCallTime_95th_percentile": 28, "TotalCallTime_98th_percentile": 29, "TotalCallTime_99th_percentile": 29, "TotalCallTime_99.9th_percentile": 29, "TotalCallTime_TimeRangeCount_0-1": 115, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 115, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 594, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 115, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 115, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 307, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 115, "sentBytes": 54860 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57068", "tag.serverName": "b60eac9bda43,36999,1732122280556", "tag.clusterId": "a3098b73-f92a-4174-8ea2-e2512d71554a", "tag.Context": "regionserver", "tag.Hostname": "b60eac9bda43", "regionCount": 13, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732122280556, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.6, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7456, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 16, "totalRowActionRequestCount": 8, "readRequestCount": 8, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 1, "Put_max": 1, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 1, "Put_75th_percentile": 1, "Put_90th_percentile": 1, "Put_95th_percentile": 1, "Put_98th_percentile": 1, "Put_99th_percentile": 1, "Put_99.9th_percentile": 1, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46694, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46694, "FlushOutputSize_max": 46694, "FlushOutputSize_mean": 46694, "FlushOutputSize_25th_percentile": 46694, "FlushOutputSize_median": 46694, "FlushOutputSize_75th_percentile": 46694, "FlushOutputSize_90th_percentile": 46694, "FlushOutputSize_95th_percentile": 46694, "FlushOutputSize_98th_percentile": 46694, "FlushOutputSize_99th_percentile": 46694, "FlushOutputSize_99.9th_percentile": 46694, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 245, "FlushTime_max": 245, "FlushTime_mean": 245, "FlushTime_25th_percentile": 245, "FlushTime_median": 245, "FlushTime_75th_percentile": 245, "FlushTime_90th_percentile": 245, "FlushTime_95th_percentile": 245, "FlushTime_98th_percentile": 245, "FlushTime_99th_percentile": 245, "FlushTime_99.9th_percentile": 245, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T17:04:49,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b60eac9bda43,36645,1732122280488 2024-11-20T17:04:49,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:04:49,146 INFO [RS:1;b60eac9bda43:36645 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:04:49,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36999-0x1015a7e0e7a0003, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b60eac9bda43,36999,1732122280556 2024-11-20T17:04:49,155 INFO [RS:2;b60eac9bda43:36999 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:04:49,164 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b60eac9bda43,36999,1732122280556] 2024-11-20T17:04:49,180 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b60eac9bda43,36999,1732122280556 already deleted, retry=false 2024-11-20T17:04:49,181 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b60eac9bda43,36999,1732122280556 expired; onlineServers=1 2024-11-20T17:04:49,181 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b60eac9bda43,36645,1732122280488] 2024-11-20T17:04:49,189 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b60eac9bda43,36645,1732122280488 already deleted, retry=false 2024-11-20T17:04:49,189 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b60eac9bda43,36645,1732122280488 expired; onlineServers=0 2024-11-20T17:04:49,189 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b60eac9bda43,44751,1732122279653' ***** 2024-11-20T17:04:49,189 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T17:04:49,189 INFO [M:0;b60eac9bda43:44751 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:04:49,189 INFO [M:0;b60eac9bda43:44751 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:04:49,189 DEBUG [M:0;b60eac9bda43:44751 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T17:04:49,189 DEBUG [M:0;b60eac9bda43:44751 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T17:04:49,189 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T17:04:49,189 DEBUG [master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.small.0-1732122281681 {}] cleaner.HFileCleaner(306): Exit Thread[master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.small.0-1732122281681,5,FailOnTimeoutGroup] 2024-11-20T17:04:49,189 DEBUG [master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.large.0-1732122281677 {}] cleaner.HFileCleaner(306): Exit Thread[master/b60eac9bda43:0:becomeActiveMaster-HFileCleaner.large.0-1732122281677,5,FailOnTimeoutGroup] 2024-11-20T17:04:49,190 INFO [M:0;b60eac9bda43:44751 {}] hbase.ChoreService(370): Chore service for: master/b60eac9bda43:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T17:04:49,190 INFO [M:0;b60eac9bda43:44751 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:04:49,190 DEBUG [M:0;b60eac9bda43:44751 {}] master.HMaster(1795): Stopping service threads 2024-11-20T17:04:49,190 INFO [M:0;b60eac9bda43:44751 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T17:04:49,190 INFO [M:0;b60eac9bda43:44751 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:04:49,190 INFO [M:0;b60eac9bda43:44751 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:04:49,190 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T17:04:49,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T17:04:49,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:04:49,197 DEBUG [M:0;b60eac9bda43:44751 {}] zookeeper.ZKUtil(347): master:44751-0x1015a7e0e7a0000, quorum=127.0.0.1:57068, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T17:04:49,197 WARN [M:0;b60eac9bda43:44751 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T17:04:49,198 INFO [M:0;b60eac9bda43:44751 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/.lastflushedseqids 2024-11-20T17:04:49,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741870_1046 (size=1492) 2024-11-20T17:04:49,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741870_1046 (size=1492) 2024-11-20T17:04:49,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741870_1046 (size=1492) 2024-11-20T17:04:49,212 INFO [M:0;b60eac9bda43:44751 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:04:49,212 INFO [M:0;b60eac9bda43:44751 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T17:04:49,212 DEBUG [M:0;b60eac9bda43:44751 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:04:49,212 INFO [M:0;b60eac9bda43:44751 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:04:49,212 DEBUG [M:0;b60eac9bda43:44751 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:04:49,212 DEBUG [M:0;b60eac9bda43:44751 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:04:49,212 DEBUG [M:0;b60eac9bda43:44751 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:04:49,212 INFO [M:0;b60eac9bda43:44751 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.17 KB heapSize=315.02 KB 2024-11-20T17:04:49,234 DEBUG [M:0;b60eac9bda43:44751 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7fa0017db4964173ab543a1bb959746f is 82, key is hbase:meta,,1/info:regioninfo/1732122287430/Put/seqid=0 2024-11-20T17:04:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33493 is added to blk_1073741871_1047 (size=6071) 2024-11-20T17:04:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41999 is added to blk_1073741871_1047 (size=6071) 2024-11-20T17:04:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741871_1047 (size=6071) 2024-11-20T17:04:49,242 INFO [M:0;b60eac9bda43:44751 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7fa0017db4964173ab543a1bb959746f 2024-11-20T17:04:49,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:04:49,264 INFO [RS:1;b60eac9bda43:36645 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:04:49,264 INFO [RS:1;b60eac9bda43:36645 {}] regionserver.HRegionServer(1031): Exiting; stopping=b60eac9bda43,36645,1732122280488; zookeeper connection closed. 2024-11-20T17:04:49,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36645-0x1015a7e0e7a0002, quorum=127.0.0.1:57068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:04:49,264 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67ad753c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67ad753c 2024-11-20T17:04:49,268 DEBUG [M:0;b60eac9bda43:44751 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34631/user/jenkins/test-data/569e3c3e-066a-04bd-987f-8e1358735e1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ab29727e896e4918acb369a6f557cad3 is 2987, key is \x00\x00\x00\x00\x00\x00\x00