2024-11-24 17:13:53,984 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b 2024-11-24 17:13:54,016 main DEBUG Took 0.029052 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 17:13:54,017 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 17:13:54,018 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 17:13:54,019 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 17:13:54,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,093 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 17:13:54,141 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,144 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,145 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,147 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,148 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,149 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,151 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,152 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,153 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,154 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,155 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,157 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 17:13:54,158 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,158 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 17:13:54,160 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 17:13:54,162 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 17:13:54,164 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 17:13:54,165 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 17:13:54,167 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 17:13:54,167 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 17:13:54,178 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 17:13:54,181 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 17:13:54,183 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 17:13:54,184 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 17:13:54,184 main DEBUG createAppenders(={Console}) 2024-11-24 17:13:54,185 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b initialized 2024-11-24 17:13:54,186 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b 2024-11-24 17:13:54,186 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b OK. 2024-11-24 17:13:54,187 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 17:13:54,187 main DEBUG OutputStream closed 2024-11-24 17:13:54,187 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 17:13:54,188 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 17:13:54,188 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK 2024-11-24 17:13:54,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 17:13:54,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 17:13:54,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 17:13:54,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 17:13:54,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 17:13:54,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 17:13:54,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 17:13:54,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 17:13:54,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 17:13:54,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 17:13:54,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 17:13:54,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 17:13:54,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 17:13:54,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 17:13:54,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 17:13:54,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 17:13:54,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 17:13:54,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 17:13:54,307 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 17:13:54,308 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@539d019) with optional ClassLoader: null 2024-11-24 17:13:54,308 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 17:13:54,309 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@539d019] started OK. 2024-11-24T17:13:54,325 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-24 17:13:54,328 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 17:13:54,328 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T17:13:54,604 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38 2024-11-24T17:13:54,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T17:13:54,651 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc, deleteOnExit=true 2024-11-24T17:13:54,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T17:13:54,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/test.cache.data in system properties and HBase conf 2024-11-24T17:13:54,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T17:13:54,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/hadoop.log.dir in system properties and HBase conf 2024-11-24T17:13:54,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T17:13:54,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T17:13:54,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T17:13:54,794 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T17:13:54,978 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T17:13:54,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T17:13:54,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T17:13:54,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T17:13:54,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T17:13:54,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T17:13:54,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T17:13:54,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T17:13:54,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T17:13:54,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T17:13:54,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/nfs.dump.dir in system properties and HBase conf 2024-11-24T17:13:54,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/java.io.tmpdir in system properties and HBase conf 2024-11-24T17:13:55,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T17:13:55,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T17:13:55,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T17:13:56,251 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T17:13:56,355 INFO [Time-limited test {}] log.Log(170): Logging initialized @4008ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T17:13:56,459 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T17:13:56,546 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T17:13:56,592 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T17:13:56,592 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T17:13:56,594 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T17:13:56,614 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T17:13:56,621 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ed58765{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/hadoop.log.dir/,AVAILABLE} 2024-11-24T17:13:56,623 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58f033c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T17:13:56,926 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59a81b67{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/java.io.tmpdir/jetty-localhost-41497-hadoop-hdfs-3_4_1-tests_jar-_-any-14164768306783170159/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T17:13:56,936 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@479cae6{HTTP/1.1, (http/1.1)}{localhost:41497} 2024-11-24T17:13:56,936 INFO [Time-limited test {}] server.Server(415): Started @4589ms 2024-11-24T17:13:57,888 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T17:13:57,903 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T17:13:57,917 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T17:13:57,917 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T17:13:57,917 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T17:13:57,918 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33a17825{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/hadoop.log.dir/,AVAILABLE} 2024-11-24T17:13:57,919 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16d70500{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T17:13:58,057 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217db90{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/java.io.tmpdir/jetty-localhost-33443-hadoop-hdfs-3_4_1-tests_jar-_-any-17776301242948524922/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T17:13:58,061 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@689992a6{HTTP/1.1, (http/1.1)}{localhost:33443} 2024-11-24T17:13:58,062 INFO [Time-limited test {}] server.Server(415): Started @5715ms 2024-11-24T17:13:58,137 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T17:13:58,293 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T17:13:58,303 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T17:13:58,304 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T17:13:58,304 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T17:13:58,305 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T17:13:58,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@529a8d1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/hadoop.log.dir/,AVAILABLE} 2024-11-24T17:13:58,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72379478{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T17:13:58,465 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7116b024{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/java.io.tmpdir/jetty-localhost-38187-hadoop-hdfs-3_4_1-tests_jar-_-any-6150811749676892098/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T17:13:58,467 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14d467cd{HTTP/1.1, (http/1.1)}{localhost:38187} 2024-11-24T17:13:58,467 INFO [Time-limited test {}] server.Server(415): Started @6120ms 2024-11-24T17:13:58,470 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T17:13:58,535 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T17:13:58,540 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T17:13:58,542 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T17:13:58,542 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T17:13:58,542 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T17:13:58,543 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@374940d7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/hadoop.log.dir/,AVAILABLE} 2024-11-24T17:13:58,544 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@449eebe4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T17:13:58,646 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10c31701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/java.io.tmpdir/jetty-localhost-43009-hadoop-hdfs-3_4_1-tests_jar-_-any-14797289170425666219/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T17:13:58,647 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b25c44{HTTP/1.1, (http/1.1)}{localhost:43009} 2024-11-24T17:13:58,647 INFO [Time-limited test {}] server.Server(415): Started @6301ms 2024-11-24T17:13:58,650 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T17:14:00,060 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data1/current/BP-1082952430-172.17.0.3-1732468435659/current, will proceed with Du for space computation calculation, 2024-11-24T17:14:00,060 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data4/current/BP-1082952430-172.17.0.3-1732468435659/current, will proceed with Du for space computation calculation, 2024-11-24T17:14:00,060 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data3/current/BP-1082952430-172.17.0.3-1732468435659/current, will proceed with Du for space computation calculation, 2024-11-24T17:14:00,060 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data2/current/BP-1082952430-172.17.0.3-1732468435659/current, will proceed with Du for space computation calculation, 2024-11-24T17:14:00,101 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T17:14:00,104 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T17:14:00,160 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd6152b7c19c1f65 with lease ID 0x5aa8ab5cd961d914: Processing first storage report for DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09 from datanode DatanodeRegistration(127.0.0.1:37919, datanodeUuid=10d72fd7-2464-4c00-b3df-47a922642fd1, infoPort=46283, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659) 2024-11-24T17:14:00,162 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd6152b7c19c1f65 with lease ID 0x5aa8ab5cd961d914: from storage DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09 node DatanodeRegistration(127.0.0.1:37919, datanodeUuid=10d72fd7-2464-4c00-b3df-47a922642fd1, infoPort=46283, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-24T17:14:00,163 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x30388405691d8633 with lease ID 0x5aa8ab5cd961d913: Processing first storage report for DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f from datanode DatanodeRegistration(127.0.0.1:45925, datanodeUuid=417ffd70-6a85-44cb-af00-bb788ba7445a, infoPort=46053, infoSecurePort=0, ipcPort=36761, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659) 2024-11-24T17:14:00,163 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x30388405691d8633 with lease ID 0x5aa8ab5cd961d913: from storage DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f node DatanodeRegistration(127.0.0.1:45925, datanodeUuid=417ffd70-6a85-44cb-af00-bb788ba7445a, infoPort=46053, infoSecurePort=0, ipcPort=36761, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T17:14:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd6152b7c19c1f65 with lease ID 0x5aa8ab5cd961d914: Processing first storage report for DS-0a519c7f-6daf-4f7e-af98-c870ca13e140 from datanode DatanodeRegistration(127.0.0.1:37919, datanodeUuid=10d72fd7-2464-4c00-b3df-47a922642fd1, infoPort=46283, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659) 2024-11-24T17:14:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd6152b7c19c1f65 with lease ID 0x5aa8ab5cd961d914: from storage DS-0a519c7f-6daf-4f7e-af98-c870ca13e140 node DatanodeRegistration(127.0.0.1:37919, datanodeUuid=10d72fd7-2464-4c00-b3df-47a922642fd1, infoPort=46283, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T17:14:00,165 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x30388405691d8633 with lease ID 0x5aa8ab5cd961d913: Processing first storage report for DS-898a2c3f-caf4-43f2-bb10-d1bc256e46a6 from datanode DatanodeRegistration(127.0.0.1:45925, datanodeUuid=417ffd70-6a85-44cb-af00-bb788ba7445a, infoPort=46053, infoSecurePort=0, ipcPort=36761, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659) 2024-11-24T17:14:00,165 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x30388405691d8633 with lease ID 0x5aa8ab5cd961d913: from storage DS-898a2c3f-caf4-43f2-bb10-d1bc256e46a6 node DatanodeRegistration(127.0.0.1:45925, datanodeUuid=417ffd70-6a85-44cb-af00-bb788ba7445a, infoPort=46053, infoSecurePort=0, ipcPort=36761, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T17:14:00,187 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data5/current/BP-1082952430-172.17.0.3-1732468435659/current, will proceed with Du for space computation calculation, 2024-11-24T17:14:00,188 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data6/current/BP-1082952430-172.17.0.3-1732468435659/current, will proceed with Du for space computation calculation, 2024-11-24T17:14:00,218 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T17:14:00,224 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1a5c9948cdbb8b4 with lease ID 0x5aa8ab5cd961d915: Processing first storage report for DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b from datanode DatanodeRegistration(127.0.0.1:37313, datanodeUuid=1900a011-dd70-46aa-9171-f1da09dda50b, infoPort=43517, infoSecurePort=0, ipcPort=35129, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659) 2024-11-24T17:14:00,225 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1a5c9948cdbb8b4 with lease ID 0x5aa8ab5cd961d915: from storage DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b node DatanodeRegistration(127.0.0.1:37313, datanodeUuid=1900a011-dd70-46aa-9171-f1da09dda50b, infoPort=43517, infoSecurePort=0, ipcPort=35129, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T17:14:00,225 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1a5c9948cdbb8b4 with lease ID 0x5aa8ab5cd961d915: Processing first storage report for DS-62c7efa8-a808-4ac7-be5e-72f43a5d8b8a from datanode DatanodeRegistration(127.0.0.1:37313, datanodeUuid=1900a011-dd70-46aa-9171-f1da09dda50b, infoPort=43517, infoSecurePort=0, ipcPort=35129, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659) 2024-11-24T17:14:00,225 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1a5c9948cdbb8b4 with lease ID 0x5aa8ab5cd961d915: from storage DS-62c7efa8-a808-4ac7-be5e-72f43a5d8b8a node DatanodeRegistration(127.0.0.1:37313, datanodeUuid=1900a011-dd70-46aa-9171-f1da09dda50b, infoPort=43517, infoSecurePort=0, ipcPort=35129, storageInfo=lv=-57;cid=testClusterID;nsid=1033329779;c=1732468435659), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T17:14:00,241 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38 2024-11-24T17:14:00,336 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/zookeeper_0, clientPort=52152, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T17:14:00,367 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52152 2024-11-24T17:14:00,384 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:00,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:00,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741825_1001 (size=7) 2024-11-24T17:14:00,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741825_1001 (size=7) 2024-11-24T17:14:00,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741825_1001 (size=7) 2024-11-24T17:14:01,130 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 with version=8 2024-11-24T17:14:01,131 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/hbase-staging 2024-11-24T17:14:01,215 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T17:14:01,444 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a23eb99f71f0:0 server-side Connection retries=45 2024-11-24T17:14:01,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:01,457 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:01,463 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T17:14:01,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:01,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T17:14:01,619 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T17:14:01,684 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T17:14:01,696 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T17:14:01,701 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T17:14:01,734 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2378 (auto-detected) 2024-11-24T17:14:01,735 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-24T17:14:01,760 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40213 2024-11-24T17:14:01,786 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40213 connecting to ZooKeeper ensemble=127.0.0.1:52152 2024-11-24T17:14:01,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402130x0, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T17:14:01,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40213-0x1016dd6789d0000 connected 2024-11-24T17:14:02,457 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:02,463 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:02,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:14:02,481 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5, hbase.cluster.distributed=false 2024-11-24T17:14:02,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T17:14:02,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40213 2024-11-24T17:14:02,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40213 2024-11-24T17:14:02,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40213 2024-11-24T17:14:02,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40213 2024-11-24T17:14:02,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40213 2024-11-24T17:14:02,638 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a23eb99f71f0:0 server-side Connection retries=45 2024-11-24T17:14:02,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,641 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T17:14:02,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T17:14:02,644 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T17:14:02,647 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T17:14:02,648 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44781 2024-11-24T17:14:02,651 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44781 connecting to ZooKeeper ensemble=127.0.0.1:52152 2024-11-24T17:14:02,652 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:02,655 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:02,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447810x0, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T17:14:02,676 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:447810x0, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:14:02,676 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44781-0x1016dd6789d0001 connected 2024-11-24T17:14:02,680 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T17:14:02,688 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T17:14:02,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T17:14:02,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T17:14:02,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44781 2024-11-24T17:14:02,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44781 2024-11-24T17:14:02,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44781 2024-11-24T17:14:02,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44781 2024-11-24T17:14:02,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44781 2024-11-24T17:14:02,719 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a23eb99f71f0:0 server-side Connection retries=45 2024-11-24T17:14:02,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,720 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T17:14:02,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T17:14:02,721 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T17:14:02,721 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T17:14:02,722 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38279 2024-11-24T17:14:02,725 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38279 connecting to ZooKeeper ensemble=127.0.0.1:52152 2024-11-24T17:14:02,726 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:02,730 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:02,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382790x0, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T17:14:02,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:382790x0, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:14:02,752 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38279-0x1016dd6789d0002 connected 2024-11-24T17:14:02,753 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T17:14:02,757 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T17:14:02,759 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T17:14:02,762 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T17:14:02,764 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38279 2024-11-24T17:14:02,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38279 2024-11-24T17:14:02,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38279 2024-11-24T17:14:02,766 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38279 2024-11-24T17:14:02,766 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38279 2024-11-24T17:14:02,786 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a23eb99f71f0:0 server-side Connection retries=45 2024-11-24T17:14:02,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,787 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T17:14:02,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T17:14:02,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T17:14:02,788 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T17:14:02,788 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T17:14:02,799 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34777 2024-11-24T17:14:02,801 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34777 connecting to ZooKeeper ensemble=127.0.0.1:52152 2024-11-24T17:14:02,802 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:02,805 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:02,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347770x0, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T17:14:02,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34777-0x1016dd6789d0003 connected 2024-11-24T17:14:02,826 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:14:02,827 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T17:14:02,832 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T17:14:02,836 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T17:14:02,838 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T17:14:02,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34777 2024-11-24T17:14:02,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34777 2024-11-24T17:14:02,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34777 2024-11-24T17:14:02,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34777 2024-11-24T17:14:02,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34777 2024-11-24T17:14:02,868 DEBUG [M:0;a23eb99f71f0:40213 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a23eb99f71f0:40213 2024-11-24T17:14:02,869 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:02,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:14:02,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:14:02,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:14:02,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:14:02,909 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:02,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T17:14:02,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T17:14:02,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T17:14:02,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:02,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:02,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:02,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:02,941 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T17:14:02,943 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a23eb99f71f0,40213,1732468441272 from backup master directory 2024-11-24T17:14:02,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:14:02,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:02,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:14:02,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:14:02,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T17:14:02,958 WARN [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T17:14:02,958 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:02,961 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T17:14:02,962 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T17:14:03,023 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/hbase.id] with ID: 866d63e0-137b-480d-bc99-aace7e6a98b3 2024-11-24T17:14:03,023 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/.tmp/hbase.id 2024-11-24T17:14:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741826_1002 (size=42) 2024-11-24T17:14:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741826_1002 (size=42) 2024-11-24T17:14:03,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741826_1002 (size=42) 2024-11-24T17:14:03,443 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/.tmp/hbase.id]:[hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/hbase.id] 2024-11-24T17:14:03,520 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T17:14:03,528 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T17:14:03,561 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 30ms. 2024-11-24T17:14:03,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:03,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:03,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:03,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:03,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741827_1003 (size=196) 2024-11-24T17:14:03,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741827_1003 (size=196) 2024-11-24T17:14:03,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741827_1003 (size=196) 2024-11-24T17:14:03,703 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:14:03,707 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T17:14:03,742 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:14:03,751 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:14:03,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741828_1004 (size=1189) 2024-11-24T17:14:03,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741828_1004 (size=1189) 2024-11-24T17:14:03,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741828_1004 (size=1189) 2024-11-24T17:14:03,899 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/data/master/store 2024-11-24T17:14:04,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741829_1005 (size=34) 2024-11-24T17:14:04,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741829_1005 (size=34) 2024-11-24T17:14:04,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741829_1005 (size=34) 2024-11-24T17:14:04,021 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T17:14:04,026 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:04,027 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T17:14:04,028 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:14:04,028 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:14:04,031 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T17:14:04,032 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:14:04,032 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:14:04,035 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732468444027Disabling compacts and flushes for region at 1732468444027Disabling writes for close at 1732468444031 (+4 ms)Writing region close event to WAL at 1732468444032 (+1 ms)Closed at 1732468444032 2024-11-24T17:14:04,042 WARN [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/data/master/store/.initializing 2024-11-24T17:14:04,043 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/WALs/a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:04,063 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T17:14:04,096 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C40213%2C1732468441272, suffix=, logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/WALs/a23eb99f71f0,40213,1732468441272, archiveDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/oldWALs, maxLogs=10 2024-11-24T17:14:04,142 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/WALs/a23eb99f71f0,40213,1732468441272/a23eb99f71f0%2C40213%2C1732468441272.1732468444106, exclude list is [], retry=0 2024-11-24T17:14:04,149 WARN [IPC Server handler 4 on default port 44865 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-24T17:14:04,149 WARN [IPC Server handler 4 on default port 44865 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-24T17:14:04,150 WARN [IPC Server handler 4 on default port 44865 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-24T17:14:04,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:04,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T17:14:04,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:04,332 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/WALs/a23eb99f71f0,40213,1732468441272/a23eb99f71f0%2C40213%2C1732468441272.1732468444106 2024-11-24T17:14:04,334 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46283:46283),(127.0.0.1/127.0.0.1:46053:46053)] 2024-11-24T17:14:04,335 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:14:04,336 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:04,343 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,346 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T17:14:04,486 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:04,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:04,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,500 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T17:14:04,501 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:04,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:04,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T17:14:04,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:04,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:04,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,524 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T17:14:04,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:04,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:04,529 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,534 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,538 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,552 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,554 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,561 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T17:14:04,568 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T17:14:04,578 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:04,580 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72446924, jitterRate=0.07954329252243042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T17:14:04,590 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732468444378Initializing all the Stores at 1732468444382 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468444383 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468444389 (+6 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468444389Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468444389Cleaning up temporary data from old regions at 1732468444554 (+165 ms)Region opened successfully at 1732468444590 (+36 ms) 2024-11-24T17:14:04,593 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T17:14:04,647 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd325e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a23eb99f71f0/172.17.0.3:0 2024-11-24T17:14:04,706 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T17:14:04,728 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:04,729 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:04,735 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:04,738 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-24T17:14:04,748 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-24T17:14:04,748 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T17:14:04,795 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T17:14:04,813 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T17:14:04,872 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T17:14:04,878 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T17:14:04,881 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T17:14:04,898 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T17:14:04,903 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T17:14:04,913 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T17:14:04,928 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T17:14:04,933 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T17:14:04,948 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T17:14:04,985 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T17:14:04,999 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T17:14:05,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T17:14:05,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T17:14:05,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T17:14:05,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T17:14:05,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,021 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a23eb99f71f0,40213,1732468441272, sessionid=0x1016dd6789d0000, setting cluster-up flag (Was=false) 2024-11-24T17:14:05,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,128 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T17:14:05,131 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:05,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:05,191 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T17:14:05,193 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:05,205 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T17:14:05,272 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(746): ClusterId : 866d63e0-137b-480d-bc99-aace7e6a98b3 2024-11-24T17:14:05,273 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(746): ClusterId : 866d63e0-137b-480d-bc99-aace7e6a98b3 2024-11-24T17:14:05,276 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(746): ClusterId : 866d63e0-137b-480d-bc99-aace7e6a98b3 2024-11-24T17:14:05,283 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T17:14:05,284 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T17:14:05,284 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T17:14:05,322 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T17:14:05,322 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T17:14:05,322 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T17:14:05,322 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T17:14:05,328 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T17:14:05,328 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T17:14:05,353 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:05,365 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T17:14:05,366 DEBUG [RS:1;a23eb99f71f0:38279 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b08b802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a23eb99f71f0/172.17.0.3:0 2024-11-24T17:14:05,368 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T17:14:05,369 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T17:14:05,370 DEBUG [RS:2;a23eb99f71f0:34777 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713e21e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a23eb99f71f0/172.17.0.3:0 2024-11-24T17:14:05,372 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T17:14:05,373 DEBUG [RS:0;a23eb99f71f0:44781 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b6b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a23eb99f71f0/172.17.0.3:0 2024-11-24T17:14:05,383 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T17:14:05,392 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a23eb99f71f0,40213,1732468441272 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T17:14:05,418 DEBUG [RS:0;a23eb99f71f0:44781 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a23eb99f71f0:44781 2024-11-24T17:14:05,419 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a23eb99f71f0:38279 2024-11-24T17:14:05,421 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a23eb99f71f0:34777 2024-11-24T17:14:05,428 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T17:14:05,428 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T17:14:05,428 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T17:14:05,428 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T17:14:05,428 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T17:14:05,428 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T17:14:05,429 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T17:14:05,429 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T17:14:05,430 DEBUG [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T17:14:05,435 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a23eb99f71f0:0, corePoolSize=5, maxPoolSize=5 2024-11-24T17:14:05,435 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a23eb99f71f0:0, corePoolSize=5, maxPoolSize=5 2024-11-24T17:14:05,436 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=5, maxPoolSize=5 2024-11-24T17:14:05,436 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=5, maxPoolSize=5 2024-11-24T17:14:05,436 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a23eb99f71f0:0, corePoolSize=10, maxPoolSize=10 2024-11-24T17:14:05,436 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:05,436 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=2, maxPoolSize=2 2024-11-24T17:14:05,437 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:05,440 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(2659): reportForDuty to master=a23eb99f71f0,40213,1732468441272 with port=38279, startcode=1732468442718 2024-11-24T17:14:05,440 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(2659): reportForDuty to master=a23eb99f71f0,40213,1732468441272 with port=34777, startcode=1732468442786 2024-11-24T17:14:05,441 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(2659): reportForDuty to master=a23eb99f71f0,40213,1732468441272 with port=44781, startcode=1732468442596 2024-11-24T17:14:05,449 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468475448 2024-11-24T17:14:05,451 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T17:14:05,452 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T17:14:05,461 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T17:14:05,462 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T17:14:05,463 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T17:14:05,463 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T17:14:05,467 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:05,467 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T17:14:05,476 DEBUG [RS:1;a23eb99f71f0:38279 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T17:14:05,478 DEBUG [RS:2;a23eb99f71f0:34777 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T17:14:05,474 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:05,478 DEBUG [RS:0;a23eb99f71f0:44781 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T17:14:05,487 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T17:14:05,500 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:05,501 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T17:14:05,504 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T17:14:05,504 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T17:14:05,510 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T17:14:05,510 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T17:14:05,516 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.large.0-1732468445512,5,FailOnTimeoutGroup] 2024-11-24T17:14:05,520 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.small.0-1732468445517,5,FailOnTimeoutGroup] 2024-11-24T17:14:05,521 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:05,521 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T17:14:05,529 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:05,530 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:05,578 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35691, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T17:14:05,578 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38021, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T17:14:05,587 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:05,591 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] master.ServerManager(517): Registering regionserver=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:05,594 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49237, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T17:14:05,631 DEBUG [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:05,632 DEBUG [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44865 2024-11-24T17:14:05,632 DEBUG [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T17:14:05,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741831_1007 (size=1321) 2024-11-24T17:14:05,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741831_1007 (size=1321) 2024-11-24T17:14:05,638 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:05,643 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] master.ServerManager(517): Registering regionserver=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:05,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741831_1007 (size=1321) 2024-11-24T17:14:05,648 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T17:14:05,649 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:05,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T17:14:05,652 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:05,652 DEBUG [RS:0;a23eb99f71f0:44781 {}] zookeeper.ZKUtil(111): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:05,653 WARN [RS:0;a23eb99f71f0:44781 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T17:14:05,653 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] master.ServerManager(517): Registering regionserver=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:05,653 INFO [RS:0;a23eb99f71f0:44781 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:14:05,653 DEBUG [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:05,654 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:05,654 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44865 2024-11-24T17:14:05,654 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T17:14:05,667 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:05,668 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44865 2024-11-24T17:14:05,668 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T17:14:05,678 WARN [IPC Server handler 3 on default port 44865 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-24T17:14:05,678 WARN [IPC Server handler 3 on default port 44865 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-24T17:14:05,679 WARN [IPC Server handler 3 on default port 44865 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-24T17:14:05,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741832_1008 (size=32) 2024-11-24T17:14:05,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741832_1008 (size=32) 2024-11-24T17:14:06,124 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a23eb99f71f0,44781,1732468442596] 2024-11-24T17:14:06,124 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a23eb99f71f0,34777,1732468442786] 2024-11-24T17:14:06,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:06,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T17:14:06,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T17:14:06,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:06,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:06,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T17:14:06,175 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T17:14:06,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:06,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:06,182 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T17:14:06,188 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T17:14:06,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:06,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:06,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T17:14:06,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T17:14:06,196 DEBUG [RS:2;a23eb99f71f0:34777 {}] zookeeper.ZKUtil(111): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:06,197 WARN [RS:2;a23eb99f71f0:34777 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T17:14:06,197 INFO [RS:2;a23eb99f71f0:34777 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:14:06,197 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:06,201 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T17:14:06,201 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:06,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:06,204 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T17:14:06,206 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a23eb99f71f0,38279,1732468442718] 2024-11-24T17:14:06,206 DEBUG [RS:1;a23eb99f71f0:38279 {}] zookeeper.ZKUtil(111): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:06,206 WARN [RS:1;a23eb99f71f0:38279 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T17:14:06,207 INFO [RS:1;a23eb99f71f0:38279 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:14:06,207 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:06,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740 2024-11-24T17:14:06,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740 2024-11-24T17:14:06,242 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T17:14:06,244 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T17:14:06,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T17:14:06,248 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T17:14:06,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T17:14:06,267 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T17:14:06,277 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T17:14:06,279 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:06,286 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59504505, jitterRate=-0.11331377923488617}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T17:14:06,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732468446148Initializing all the Stores at 1732468446152 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468446152Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468446152Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468446153 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468446153Cleaning up temporary data from old regions at 1732468446245 (+92 ms)Region opened successfully at 1732468446287 (+42 ms) 2024-11-24T17:14:06,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T17:14:06,288 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T17:14:06,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T17:14:06,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T17:14:06,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T17:14:06,290 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T17:14:06,291 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732468446288Disabling compacts and flushes for region at 1732468446288Disabling writes for close at 1732468446288Writing region close event to WAL at 1732468446289 (+1 ms)Closed at 1732468446290 (+1 ms) 2024-11-24T17:14:06,299 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:06,299 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T17:14:06,299 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T17:14:06,302 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T17:14:06,303 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T17:14:06,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T17:14:06,321 INFO [RS:1;a23eb99f71f0:38279 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T17:14:06,321 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,327 INFO [RS:2;a23eb99f71f0:34777 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T17:14:06,328 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,333 INFO [RS:0;a23eb99f71f0:44781 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T17:14:06,335 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,338 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T17:14:06,338 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T17:14:06,341 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T17:14:06,359 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T17:14:06,361 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T17:14:06,362 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:06,360 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T17:14:06,369 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,369 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,369 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,369 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0, corePoolSize=2, maxPoolSize=2 2024-11-24T17:14:06,370 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0, corePoolSize=2, maxPoolSize=2 2024-11-24T17:14:06,370 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,370 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a23eb99f71f0:0, corePoolSize=3, maxPoolSize=3 2024-11-24T17:14:06,371 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,371 DEBUG [RS:0;a23eb99f71f0:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a23eb99f71f0:0, corePoolSize=3, maxPoolSize=3 2024-11-24T17:14:06,371 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a23eb99f71f0:0, corePoolSize=3, maxPoolSize=3 2024-11-24T17:14:06,371 DEBUG [RS:2;a23eb99f71f0:34777 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a23eb99f71f0:0, corePoolSize=3, maxPoolSize=3 2024-11-24T17:14:06,372 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,372 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,372 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,373 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,373 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,373 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,373 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0, corePoolSize=2, maxPoolSize=2 2024-11-24T17:14:06,373 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,373 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,373 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,374 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,374 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,374 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a23eb99f71f0:0, corePoolSize=1, maxPoolSize=1 2024-11-24T17:14:06,374 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a23eb99f71f0:0, corePoolSize=3, maxPoolSize=3 2024-11-24T17:14:06,374 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T17:14:06,374 DEBUG [RS:1;a23eb99f71f0:38279 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a23eb99f71f0:0, corePoolSize=3, maxPoolSize=3 2024-11-24T17:14:06,388 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,388 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,389 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,389 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,389 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,389 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,34777,1732468442786-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T17:14:06,453 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T17:14:06,456 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,34777,1732468442786-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,462 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,463 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.Replication(171): a23eb99f71f0,34777,1732468442786 started 2024-11-24T17:14:06,464 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,464 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,465 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,465 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,465 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,465 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,38279,1732468442718-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T17:14:06,487 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,488 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,488 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,488 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,489 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,489 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,44781,1732468442596-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T17:14:06,498 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T17:14:06,499 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,38279,1732468442718-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,499 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,499 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.Replication(171): a23eb99f71f0,38279,1732468442718 started 2024-11-24T17:14:06,510 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,511 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1482): Serving as a23eb99f71f0,34777,1732468442786, RpcServer on a23eb99f71f0/172.17.0.3:34777, sessionid=0x1016dd6789d0003 2024-11-24T17:14:06,515 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T17:14:06,515 DEBUG [RS:2;a23eb99f71f0:34777 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:06,516 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a23eb99f71f0,34777,1732468442786' 2024-11-24T17:14:06,516 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T17:14:06,522 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T17:14:06,525 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T17:14:06,525 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T17:14:06,526 DEBUG [RS:2;a23eb99f71f0:34777 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:06,526 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a23eb99f71f0,34777,1732468442786' 2024-11-24T17:14:06,526 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T17:14:06,529 WARN [a23eb99f71f0:40213 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T17:14:06,536 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,537 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1482): Serving as a23eb99f71f0,38279,1732468442718, RpcServer on a23eb99f71f0/172.17.0.3:38279, sessionid=0x1016dd6789d0002 2024-11-24T17:14:06,537 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T17:14:06,537 DEBUG [RS:1;a23eb99f71f0:38279 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:06,537 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a23eb99f71f0,38279,1732468442718' 2024-11-24T17:14:06,537 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T17:14:06,537 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T17:14:06,545 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T17:14:06,545 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,44781,1732468442596-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,546 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,546 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.Replication(171): a23eb99f71f0,44781,1732468442596 started 2024-11-24T17:14:06,549 DEBUG [RS:2;a23eb99f71f0:34777 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T17:14:06,549 INFO [RS:2;a23eb99f71f0:34777 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T17:14:06,550 INFO [RS:2;a23eb99f71f0:34777 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T17:14:06,551 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T17:14:06,564 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T17:14:06,564 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T17:14:06,565 DEBUG [RS:1;a23eb99f71f0:38279 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:06,565 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a23eb99f71f0,38279,1732468442718' 2024-11-24T17:14:06,565 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T17:14:06,580 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T17:14:06,583 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:06,583 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(1482): Serving as a23eb99f71f0,44781,1732468442596, RpcServer on a23eb99f71f0/172.17.0.3:44781, sessionid=0x1016dd6789d0001 2024-11-24T17:14:06,585 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T17:14:06,585 DEBUG [RS:0;a23eb99f71f0:44781 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:06,585 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a23eb99f71f0,44781,1732468442596' 2024-11-24T17:14:06,585 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T17:14:06,596 DEBUG [RS:1;a23eb99f71f0:38279 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T17:14:06,597 INFO [RS:1;a23eb99f71f0:38279 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T17:14:06,597 INFO [RS:1;a23eb99f71f0:38279 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T17:14:06,599 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T17:14:06,612 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T17:14:06,612 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T17:14:06,612 DEBUG [RS:0;a23eb99f71f0:44781 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:06,613 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a23eb99f71f0,44781,1732468442596' 2024-11-24T17:14:06,613 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T17:14:06,622 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T17:14:06,627 DEBUG [RS:0;a23eb99f71f0:44781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T17:14:06,627 INFO [RS:0;a23eb99f71f0:44781 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T17:14:06,627 INFO [RS:0;a23eb99f71f0:44781 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T17:14:06,683 INFO [RS:2;a23eb99f71f0:34777 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T17:14:06,698 INFO [RS:1;a23eb99f71f0:38279 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T17:14:06,699 INFO [RS:2;a23eb99f71f0:34777 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C34777%2C1732468442786, suffix=, logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,34777,1732468442786, archiveDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs, maxLogs=32 2024-11-24T17:14:06,721 INFO [RS:1;a23eb99f71f0:38279 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C38279%2C1732468442718, suffix=, logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718, archiveDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs, maxLogs=32 2024-11-24T17:14:06,734 INFO [RS:0;a23eb99f71f0:44781 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T17:14:06,749 INFO [RS:0;a23eb99f71f0:44781 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C44781%2C1732468442596, suffix=, logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596, archiveDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs, maxLogs=32 2024-11-24T17:14:06,754 DEBUG [RS:2;a23eb99f71f0:34777 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,34777,1732468442786/a23eb99f71f0%2C34777%2C1732468442786.1732468446703, exclude list is [], retry=0 2024-11-24T17:14:06,765 DEBUG [RS:1;a23eb99f71f0:38279 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718/a23eb99f71f0%2C38279%2C1732468442718.1732468446724, exclude list is [], retry=0 2024-11-24T17:14:06,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:06,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK] 2024-11-24T17:14:06,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:06,821 DEBUG [RS:0;a23eb99f71f0:44781 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.1732468446752, exclude list is [], retry=0 2024-11-24T17:14:06,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:06,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK] 2024-11-24T17:14:06,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:06,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:06,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK] 2024-11-24T17:14:06,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:06,888 INFO [RS:2;a23eb99f71f0:34777 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,34777,1732468442786/a23eb99f71f0%2C34777%2C1732468442786.1732468446703 2024-11-24T17:14:06,895 DEBUG [RS:2;a23eb99f71f0:34777 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43517:43517),(127.0.0.1/127.0.0.1:46053:46053),(127.0.0.1/127.0.0.1:46283:46283)] 2024-11-24T17:14:06,909 INFO [RS:1;a23eb99f71f0:38279 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718/a23eb99f71f0%2C38279%2C1732468442718.1732468446724 2024-11-24T17:14:06,909 INFO [RS:0;a23eb99f71f0:44781 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.1732468446752 2024-11-24T17:14:06,916 DEBUG [RS:1;a23eb99f71f0:38279 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46283:46283),(127.0.0.1/127.0.0.1:43517:43517),(127.0.0.1/127.0.0.1:46053:46053)] 2024-11-24T17:14:06,925 DEBUG [RS:0;a23eb99f71f0:44781 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46283:46283),(127.0.0.1/127.0.0.1:43517:43517),(127.0.0.1/127.0.0.1:46053:46053)] 2024-11-24T17:14:07,035 DEBUG [a23eb99f71f0:40213 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T17:14:07,050 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(204): Hosts are {a23eb99f71f0=0} racks are {/default-rack=0} 2024-11-24T17:14:07,081 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T17:14:07,081 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T17:14:07,082 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T17:14:07,082 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T17:14:07,082 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T17:14:07,082 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T17:14:07,082 INFO [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T17:14:07,082 INFO [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T17:14:07,082 INFO [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T17:14:07,082 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T17:14:07,095 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:07,113 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,44781,1732468442596, state=OPENING 2024-11-24T17:14:07,465 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T17:14:07,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:07,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:07,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:07,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:07,786 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:07,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:07,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:07,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:07,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:07,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:07,993 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T17:14:07,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55111, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T17:14:08,034 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T17:14:08,035 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:14:08,036 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T17:14:08,053 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C44781%2C1732468442596.meta, suffix=.meta, logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596, archiveDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs, maxLogs=32 2024-11-24T17:14:08,079 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, exclude list is [], retry=0 2024-11-24T17:14:08,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK] 2024-11-24T17:14:08,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:08,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:08,121 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta 2024-11-24T17:14:08,123 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43517:43517),(127.0.0.1/127.0.0.1:46283:46283),(127.0.0.1/127.0.0.1:46053:46053)] 2024-11-24T17:14:08,124 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:14:08,127 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T17:14:08,133 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T17:14:08,140 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T17:14:08,146 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T17:14:08,147 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:08,150 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T17:14:08,154 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T17:14:08,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T17:14:08,194 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T17:14:08,195 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:08,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:08,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T17:14:08,214 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T17:14:08,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:08,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:08,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T17:14:08,236 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T17:14:08,236 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:08,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:08,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T17:14:08,251 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T17:14:08,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:08,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:08,263 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T17:14:08,277 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740 2024-11-24T17:14:08,304 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740 2024-11-24T17:14:08,328 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T17:14:08,329 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T17:14:08,334 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T17:14:08,345 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T17:14:08,352 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59174510, jitterRate=-0.11823108792304993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T17:14:08,352 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T17:14:08,355 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732468448155Writing region info on filesystem at 1732468448155Initializing all the Stores at 1732468448166 (+11 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468448166Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468448173 (+7 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468448173Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468448173Cleaning up temporary data from old regions at 1732468448329 (+156 ms)Running coprocessor post-open hooks at 1732468448353 (+24 ms)Region opened successfully at 1732468448354 (+1 ms) 2024-11-24T17:14:08,373 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732468447979 2024-11-24T17:14:08,413 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T17:14:08,414 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T17:14:08,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:08,431 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,44781,1732468442596, state=OPEN 2024-11-24T17:14:08,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:08,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:08,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:08,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:08,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:08,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:08,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:08,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:08,918 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:08,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T17:14:08,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,44781,1732468442596 in 1.1290 sec 2024-11-24T17:14:08,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T17:14:08,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 2.6250 sec 2024-11-24T17:14:08,951 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:08,951 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T17:14:08,987 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T17:14:08,989 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1] 2024-11-24T17:14:09,023 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T17:14:09,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46579, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T17:14:09,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 3.8180 sec 2024-11-24T17:14:09,092 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732468449092, completionTime=-1 2024-11-24T17:14:09,096 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T17:14:09,097 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:09,269 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T17:14:09,270 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468509269 2024-11-24T17:14:09,270 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468569270 2024-11-24T17:14:09,270 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 173 msec 2024-11-24T17:14:09,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741832_1008 (size=32) 2024-11-24T17:14:09,274 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:09,300 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,40213,1732468441272-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:09,300 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,40213,1732468441272-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:09,300 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,40213,1732468441272-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:09,302 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a23eb99f71f0:40213, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:09,303 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:09,307 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:09,318 DEBUG [master/a23eb99f71f0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T17:14:09,358 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 6.400sec 2024-11-24T17:14:09,361 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T17:14:09,363 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T17:14:09,367 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T17:14:09,368 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T17:14:09,368 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T17:14:09,370 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,40213,1732468441272-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T17:14:09,371 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,40213,1732468441272-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T17:14:09,396 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T17:14:09,398 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T17:14:09,398 INFO [master/a23eb99f71f0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a23eb99f71f0,40213,1732468441272-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T17:14:09,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f59a452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:14:09,458 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T17:14:09,458 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T17:14:09,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a23eb99f71f0,40213,-1 for getting cluster id 2024-11-24T17:14:09,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T17:14:09,501 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '866d63e0-137b-480d-bc99-aace7e6a98b3' 2024-11-24T17:14:09,505 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T17:14:09,505 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "866d63e0-137b-480d-bc99-aace7e6a98b3" 2024-11-24T17:14:09,506 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52db5960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:14:09,506 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a23eb99f71f0,40213,-1] 2024-11-24T17:14:09,509 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T17:14:09,514 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:09,519 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T17:14:09,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5f57e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T17:14:09,527 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T17:14:09,550 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1] 2024-11-24T17:14:09,550 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T17:14:09,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T17:14:09,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:09,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T17:14:09,633 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:09,640 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7dec82e6 2024-11-24T17:14:09,642 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T17:14:09,652 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T17:14:09,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T17:14:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:09,715 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T17:14:09,720 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:09,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-24T17:14:09,728 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T17:14:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:09,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741837_1013 (size=358) 2024-11-24T17:14:09,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741837_1013 (size=358) 2024-11-24T17:14:09,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741837_1013 (size=358) 2024-11-24T17:14:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:10,241 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d888fda423b365786e1d269df49d1cfd, NAME => 'test,,1732468449655.d888fda423b365786e1d269df49d1cfd.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,242 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 94439407a643ae820e9015188cf23ed6, NAME => 'test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,248 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4b5321b4dd571156ebac77ca1fa8b150, NAME => 'test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741839_1015 (size=45) 2024-11-24T17:14:10,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741838_1014 (size=42) 2024-11-24T17:14:10,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741839_1015 (size=45) 2024-11-24T17:14:10,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741838_1014 (size=42) 2024-11-24T17:14:10,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741838_1014 (size=42) 2024-11-24T17:14:10,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741839_1015 (size=45) 2024-11-24T17:14:10,326 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741840_1016 (size=45) 2024-11-24T17:14:10,327 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 94439407a643ae820e9015188cf23ed6, disabling compactions & flushes 2024-11-24T17:14:10,327 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:10,327 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:10,327 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. after waiting 0 ms 2024-11-24T17:14:10,327 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:10,327 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:10,328 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 94439407a643ae820e9015188cf23ed6: Waiting for close lock at 1732468450327Disabling compacts and flushes for region at 1732468450327Disabling writes for close at 1732468450327Writing region close event to WAL at 1732468450327Closed at 1732468450327 2024-11-24T17:14:10,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741840_1016 (size=45) 2024-11-24T17:14:10,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741840_1016 (size=45) 2024-11-24T17:14:10,332 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4a1b996a2c1476427750ae2fbe015bc5, NAME => 'test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,332 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732468449655.d888fda423b365786e1d269df49d1cfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,333 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,333 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4b5321b4dd571156ebac77ca1fa8b150, disabling compactions & flushes 2024-11-24T17:14:10,333 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. 2024-11-24T17:14:10,333 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. 2024-11-24T17:14:10,333 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. after waiting 0 ms 2024-11-24T17:14:10,333 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. 2024-11-24T17:14:10,333 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. 2024-11-24T17:14:10,333 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4b5321b4dd571156ebac77ca1fa8b150: Waiting for close lock at 1732468450333Disabling compacts and flushes for region at 1732468450333Disabling writes for close at 1732468450333Writing region close event to WAL at 1732468450333Closed at 1732468450333 2024-11-24T17:14:10,334 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e88c3c1672dcd6a9395b368267add589, NAME => 'test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,332 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d888fda423b365786e1d269df49d1cfd, disabling compactions & flushes 2024-11-24T17:14:10,335 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:10,335 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:10,335 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732468449655.d888fda423b365786e1d269df49d1cfd. after waiting 0 ms 2024-11-24T17:14:10,335 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:10,335 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:10,335 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d888fda423b365786e1d269df49d1cfd: Waiting for close lock at 1732468450332Disabling compacts and flushes for region at 1732468450332Disabling writes for close at 1732468450335 (+3 ms)Writing region close event to WAL at 1732468450335Closed at 1732468450335 2024-11-24T17:14:10,336 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 448f6154a795fbeb01b36798bcbacb8f, NAME => 'test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:10,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741841_1017 (size=45) 2024-11-24T17:14:10,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741841_1017 (size=45) 2024-11-24T17:14:10,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741841_1017 (size=45) 2024-11-24T17:14:10,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 448f6154a795fbeb01b36798bcbacb8f, disabling compactions & flushes 2024-11-24T17:14:10,402 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:10,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:10,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. after waiting 0 ms 2024-11-24T17:14:10,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:10,402 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:10,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 448f6154a795fbeb01b36798bcbacb8f: Waiting for close lock at 1732468450402Disabling compacts and flushes for region at 1732468450402Disabling writes for close at 1732468450402Writing region close event to WAL at 1732468450402Closed at 1732468450402 2024-11-24T17:14:10,404 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4707e344fe9a8cd93af441dfe2b9492e, NAME => 'test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741843_1019 (size=45) 2024-11-24T17:14:10,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741843_1019 (size=45) 2024-11-24T17:14:10,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741843_1019 (size=45) 2024-11-24T17:14:10,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741842_1018 (size=45) 2024-11-24T17:14:10,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741842_1018 (size=45) 2024-11-24T17:14:10,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741842_1018 (size=45) 2024-11-24T17:14:10,432 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4a1b996a2c1476427750ae2fbe015bc5, disabling compactions & flushes 2024-11-24T17:14:10,433 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:10,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:10,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. after waiting 0 ms 2024-11-24T17:14:10,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:10,433 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:10,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4a1b996a2c1476427750ae2fbe015bc5: Waiting for close lock at 1732468450433Disabling compacts and flushes for region at 1732468450433Disabling writes for close at 1732468450433Writing region close event to WAL at 1732468450433Closed at 1732468450433 2024-11-24T17:14:10,434 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 89f671091abaef7d04c744da188c0bf9, NAME => 'test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741844_1020 (size=45) 2024-11-24T17:14:10,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741844_1020 (size=45) 2024-11-24T17:14:10,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741844_1020 (size=45) 2024-11-24T17:14:10,497 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,497 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4707e344fe9a8cd93af441dfe2b9492e, disabling compactions & flushes 2024-11-24T17:14:10,498 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:10,498 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:10,498 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. after waiting 0 ms 2024-11-24T17:14:10,498 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:10,498 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:10,498 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4707e344fe9a8cd93af441dfe2b9492e: Waiting for close lock at 1732468450497Disabling compacts and flushes for region at 1732468450497Disabling writes for close at 1732468450498 (+1 ms)Writing region close event to WAL at 1732468450498Closed at 1732468450498 2024-11-24T17:14:10,499 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3963c202e600d921e58afb7a1966f4f0, NAME => 'test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741845_1021 (size=45) 2024-11-24T17:14:10,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741845_1021 (size=45) 2024-11-24T17:14:10,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741845_1021 (size=45) 2024-11-24T17:14:10,514 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,515 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 89f671091abaef7d04c744da188c0bf9, disabling compactions & flushes 2024-11-24T17:14:10,515 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:10,515 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:10,515 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. after waiting 0 ms 2024-11-24T17:14:10,515 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:10,515 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:10,515 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 89f671091abaef7d04c744da188c0bf9: Waiting for close lock at 1732468450515Disabling compacts and flushes for region at 1732468450515Disabling writes for close at 1732468450515Writing region close event to WAL at 1732468450515Closed at 1732468450515 2024-11-24T17:14:10,516 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3132e986d91cd07c924412b1164b0ada, NAME => 'test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741846_1022 (size=45) 2024-11-24T17:14:10,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741846_1022 (size=45) 2024-11-24T17:14:10,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741846_1022 (size=45) 2024-11-24T17:14:10,574 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,574 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3963c202e600d921e58afb7a1966f4f0, disabling compactions & flushes 2024-11-24T17:14:10,574 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:10,574 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:10,574 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. after waiting 0 ms 2024-11-24T17:14:10,574 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:10,574 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:10,575 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3963c202e600d921e58afb7a1966f4f0: Waiting for close lock at 1732468450574Disabling compacts and flushes for region at 1732468450574Disabling writes for close at 1732468450574Writing region close event to WAL at 1732468450574Closed at 1732468450574 2024-11-24T17:14:10,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741847_1023 (size=45) 2024-11-24T17:14:10,577 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => acf1021fd9104e08951bd81c1946b074, NAME => 'test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:10,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3132e986d91cd07c924412b1164b0ada, disabling compactions & flushes 2024-11-24T17:14:10,579 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:10,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741847_1023 (size=45) 2024-11-24T17:14:10,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:10,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. after waiting 0 ms 2024-11-24T17:14:10,579 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:10,580 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:10,580 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3132e986d91cd07c924412b1164b0ada: Waiting for close lock at 1732468450579Disabling compacts and flushes for region at 1732468450579Disabling writes for close at 1732468450579Writing region close event to WAL at 1732468450580 (+1 ms)Closed at 1732468450580 2024-11-24T17:14:10,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741847_1023 (size=45) 2024-11-24T17:14:10,580 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a61743f7cd3883457b967d55f98541e1, NAME => 'test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:10,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741849_1025 (size=45) 2024-11-24T17:14:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741849_1025 (size=45) 2024-11-24T17:14:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741848_1024 (size=45) 2024-11-24T17:14:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741848_1024 (size=45) 2024-11-24T17:14:10,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741849_1025 (size=45) 2024-11-24T17:14:10,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741848_1024 (size=45) 2024-11-24T17:14:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:11,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T17:14:11,680 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T17:14:11,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T17:14:11,689 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T17:14:11,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T17:14:11,689 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T17:14:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:12,228 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T17:14:13,682 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:14:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a61743f7cd3883457b967d55f98541e1, disabling compactions & flushes 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing acf1021fd9104e08951bd81c1946b074, disabling compactions & flushes 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:16,845 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:16,845 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e88c3c1672dcd6a9395b368267add589, disabling compactions & flushes 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. after waiting 0 ms 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. after waiting 0 ms 2024-11-24T17:14:16,845 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:16,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:16,846 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. after waiting 0 ms 2024-11-24T17:14:16,846 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:16,846 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:16,846 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:16,846 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a61743f7cd3883457b967d55f98541e1: Waiting for close lock at 1732468456845Disabling compacts and flushes for region at 1732468456845Disabling writes for close at 1732468456845Writing region close event to WAL at 1732468456845Closed at 1732468456846 (+1 ms) 2024-11-24T17:14:16,846 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e88c3c1672dcd6a9395b368267add589: Waiting for close lock at 1732468456845Disabling compacts and flushes for region at 1732468456845Disabling writes for close at 1732468456846 (+1 ms)Writing region close event to WAL at 1732468456846Closed at 1732468456846 2024-11-24T17:14:16,846 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 12eb6cfa7992f8b2d57701a44d5e5526, NAME => 'test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:16,847 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:16,847 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for acf1021fd9104e08951bd81c1946b074: Waiting for close lock at 1732468456845Disabling compacts and flushes for region at 1732468456845Disabling writes for close at 1732468456845Writing region close event to WAL at 1732468456845Closed at 1732468456847 (+2 ms) 2024-11-24T17:14:16,847 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b1ebd571dd5d321046e5067f5b19ad97, NAME => 'test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:16,848 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 68cc51974c64ffdbd8bb6b53a87450da, NAME => 'test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:20,449 WARN [WorkerMonitor {}] procedure2.ProcedureExecutor$WorkerMonitor(2269): Worker stuck PEWorker-1(pid=4), run time 10.7530 sec 2024-11-24T17:14:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741852_1028 (size=45) 2024-11-24T17:14:24,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741852_1028 (size=45) 2024-11-24T17:14:24,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741852_1028 (size=45) 2024-11-24T17:14:24,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741851_1027 (size=45) 2024-11-24T17:14:24,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b1ebd571dd5d321046e5067f5b19ad97, disabling compactions & flushes 2024-11-24T17:14:24,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741851_1027 (size=45) 2024-11-24T17:14:24,162 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:24,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:24,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. after waiting 0 ms 2024-11-24T17:14:24,163 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:24,163 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:24,163 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b1ebd571dd5d321046e5067f5b19ad97: Waiting for close lock at 1732468464162Disabling compacts and flushes for region at 1732468464162Disabling writes for close at 1732468464163 (+1 ms)Writing region close event to WAL at 1732468464163Closed at 1732468464163 2024-11-24T17:14:24,164 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3bf176f82dd506ad26a9f1d6ebd6f33a, NAME => 'test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741851_1027 (size=45) 2024-11-24T17:14:24,171 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,171 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 12eb6cfa7992f8b2d57701a44d5e5526, disabling compactions & flushes 2024-11-24T17:14:24,171 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:24,171 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:24,171 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. after waiting 0 ms 2024-11-24T17:14:24,172 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:24,172 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:24,172 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 12eb6cfa7992f8b2d57701a44d5e5526: Waiting for close lock at 1732468464171Disabling compacts and flushes for region at 1732468464171Disabling writes for close at 1732468464171Writing region close event to WAL at 1732468464172 (+1 ms)Closed at 1732468464172 2024-11-24T17:14:24,172 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a6d2969d9f925c5b8cb0ff244090dcd4, NAME => 'test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741850_1026 (size=45) 2024-11-24T17:14:24,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741850_1026 (size=45) 2024-11-24T17:14:24,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741850_1026 (size=45) 2024-11-24T17:14:24,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 68cc51974c64ffdbd8bb6b53a87450da, disabling compactions & flushes 2024-11-24T17:14:24,199 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. after waiting 0 ms 2024-11-24T17:14:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:24,199 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 68cc51974c64ffdbd8bb6b53a87450da: Waiting for close lock at 1732468464198Disabling compacts and flushes for region at 1732468464198Disabling writes for close at 1732468464199 (+1 ms)Writing region close event to WAL at 1732468464199Closed at 1732468464199 2024-11-24T17:14:24,200 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ba4002d0b6663684be2c126a81b2c74, NAME => 'test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741853_1029 (size=45) 2024-11-24T17:14:24,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741853_1029 (size=45) 2024-11-24T17:14:24,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741853_1029 (size=45) 2024-11-24T17:14:24,268 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,268 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 3bf176f82dd506ad26a9f1d6ebd6f33a, disabling compactions & flushes 2024-11-24T17:14:24,268 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:24,269 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:24,269 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. after waiting 0 ms 2024-11-24T17:14:24,269 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:24,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741854_1030 (size=45) 2024-11-24T17:14:24,271 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:24,271 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3bf176f82dd506ad26a9f1d6ebd6f33a: Waiting for close lock at 1732468464268Disabling compacts and flushes for region at 1732468464268Disabling writes for close at 1732468464269 (+1 ms)Writing region close event to WAL at 1732468464269Closed at 1732468464269 2024-11-24T17:14:24,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741854_1030 (size=45) 2024-11-24T17:14:24,272 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4ed93b4907aeb35b406e6de08e103616, NAME => 'test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a6d2969d9f925c5b8cb0ff244090dcd4, disabling compactions & flushes 2024-11-24T17:14:24,275 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:24,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:24,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. after waiting 0 ms 2024-11-24T17:14:24,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:24,275 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:24,275 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a6d2969d9f925c5b8cb0ff244090dcd4: Waiting for close lock at 1732468464274Disabling compacts and flushes for region at 1732468464274Disabling writes for close at 1732468464275 (+1 ms)Writing region close event to WAL at 1732468464275Closed at 1732468464275 2024-11-24T17:14:24,276 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6f16dc15c0e0aff7efc028181a4fdbe3, NAME => 'test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741854_1030 (size=45) 2024-11-24T17:14:24,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741855_1031 (size=45) 2024-11-24T17:14:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741855_1031 (size=45) 2024-11-24T17:14:24,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741855_1031 (size=45) 2024-11-24T17:14:24,310 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,310 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7ba4002d0b6663684be2c126a81b2c74, disabling compactions & flushes 2024-11-24T17:14:24,310 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:24,310 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:24,310 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. after waiting 0 ms 2024-11-24T17:14:24,310 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:24,310 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:24,310 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ba4002d0b6663684be2c126a81b2c74: Waiting for close lock at 1732468464310Disabling compacts and flushes for region at 1732468464310Disabling writes for close at 1732468464310Writing region close event to WAL at 1732468464310Closed at 1732468464310 2024-11-24T17:14:24,311 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 93d787842f2c5c53adaf00e6e90673e7, NAME => 'test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741858_1034 (size=45) 2024-11-24T17:14:24,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741858_1034 (size=45) 2024-11-24T17:14:24,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741858_1034 (size=45) 2024-11-24T17:14:24,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741856_1032 (size=45) 2024-11-24T17:14:24,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741856_1032 (size=45) 2024-11-24T17:14:24,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741856_1032 (size=45) 2024-11-24T17:14:24,387 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,387 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4ed93b4907aeb35b406e6de08e103616, disabling compactions & flushes 2024-11-24T17:14:24,387 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:24,387 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:24,387 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. after waiting 0 ms 2024-11-24T17:14:24,387 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:24,387 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:24,387 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4ed93b4907aeb35b406e6de08e103616: Waiting for close lock at 1732468464387Disabling compacts and flushes for region at 1732468464387Disabling writes for close at 1732468464387Writing region close event to WAL at 1732468464387Closed at 1732468464387 2024-11-24T17:14:24,388 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 71e89563154e9bf392a9193da474463d, NAME => 'test,vvv,1732468449655.71e89563154e9bf392a9193da474463d.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741857_1033 (size=45) 2024-11-24T17:14:24,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741857_1033 (size=45) 2024-11-24T17:14:24,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741857_1033 (size=45) 2024-11-24T17:14:24,408 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6f16dc15c0e0aff7efc028181a4fdbe3, disabling compactions & flushes 2024-11-24T17:14:24,409 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:24,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:24,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. after waiting 0 ms 2024-11-24T17:14:24,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:24,409 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:24,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6f16dc15c0e0aff7efc028181a4fdbe3: Waiting for close lock at 1732468464409Disabling compacts and flushes for region at 1732468464409Disabling writes for close at 1732468464409Writing region close event to WAL at 1732468464409Closed at 1732468464409 2024-11-24T17:14:24,410 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 67115580ab5444880122d14f983db95c, NAME => 'test,www,1732468449655.67115580ab5444880122d14f983db95c.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741859_1035 (size=45) 2024-11-24T17:14:24,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741859_1035 (size=45) 2024-11-24T17:14:24,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741859_1035 (size=45) 2024-11-24T17:14:24,434 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732468449655.71e89563154e9bf392a9193da474463d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,434 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 71e89563154e9bf392a9193da474463d, disabling compactions & flushes 2024-11-24T17:14:24,435 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:24,435 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:24,435 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. after waiting 0 ms 2024-11-24T17:14:24,435 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:24,435 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:24,435 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 71e89563154e9bf392a9193da474463d: Waiting for close lock at 1732468464434Disabling compacts and flushes for region at 1732468464434Disabling writes for close at 1732468464435 (+1 ms)Writing region close event to WAL at 1732468464435Closed at 1732468464435 2024-11-24T17:14:24,436 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 58c2a35cef97d825f565d67ca7006c36, NAME => 'test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741860_1036 (size=45) 2024-11-24T17:14:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741860_1036 (size=45) 2024-11-24T17:14:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741860_1036 (size=45) 2024-11-24T17:14:24,460 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1732468449655.67115580ab5444880122d14f983db95c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,460 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 67115580ab5444880122d14f983db95c, disabling compactions & flushes 2024-11-24T17:14:24,460 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:24,460 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:24,460 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732468449655.67115580ab5444880122d14f983db95c. after waiting 0 ms 2024-11-24T17:14:24,460 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:24,460 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:24,461 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 67115580ab5444880122d14f983db95c: Waiting for close lock at 1732468464460Disabling compacts and flushes for region at 1732468464460Disabling writes for close at 1732468464460Writing region close event to WAL at 1732468464460Closed at 1732468464460 2024-11-24T17:14:24,461 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 61f230c444c764032b318ed0a81000ab, NAME => 'test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741861_1037 (size=45) 2024-11-24T17:14:24,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741861_1037 (size=45) 2024-11-24T17:14:24,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741861_1037 (size=45) 2024-11-24T17:14:24,509 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,509 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 58c2a35cef97d825f565d67ca7006c36, disabling compactions & flushes 2024-11-24T17:14:24,509 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:24,509 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:24,509 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. after waiting 0 ms 2024-11-24T17:14:24,509 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:24,509 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:24,509 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 58c2a35cef97d825f565d67ca7006c36: Waiting for close lock at 1732468464509Disabling compacts and flushes for region at 1732468464509Disabling writes for close at 1732468464509Writing region close event to WAL at 1732468464509Closed at 1732468464509 2024-11-24T17:14:24,510 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4f35eb03f4e248bda613971e436befd3, NAME => 'test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5 2024-11-24T17:14:24,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741862_1038 (size=45) 2024-11-24T17:14:24,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741862_1038 (size=45) 2024-11-24T17:14:24,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741862_1038 (size=45) 2024-11-24T17:14:24,562 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,562 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 61f230c444c764032b318ed0a81000ab, disabling compactions & flushes 2024-11-24T17:14:24,562 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:24,562 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:24,562 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. after waiting 0 ms 2024-11-24T17:14:24,562 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:24,562 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:24,562 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 61f230c444c764032b318ed0a81000ab: Waiting for close lock at 1732468464562Disabling compacts and flushes for region at 1732468464562Disabling writes for close at 1732468464562Writing region close event to WAL at 1732468464562Closed at 1732468464562 2024-11-24T17:14:24,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741863_1039 (size=42) 2024-11-24T17:14:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741863_1039 (size=42) 2024-11-24T17:14:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741863_1039 (size=42) 2024-11-24T17:14:24,590 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,590 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4f35eb03f4e248bda613971e436befd3, disabling compactions & flushes 2024-11-24T17:14:24,590 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:24,590 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:24,590 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. after waiting 0 ms 2024-11-24T17:14:24,590 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:24,590 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:24,590 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4f35eb03f4e248bda613971e436befd3: Waiting for close lock at 1732468464590Disabling compacts and flushes for region at 1732468464590Disabling writes for close at 1732468464590Writing region close event to WAL at 1732468464590Closed at 1732468464590 2024-11-24T17:14:24,782 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:24,782 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 93d787842f2c5c53adaf00e6e90673e7, disabling compactions & flushes 2024-11-24T17:14:24,783 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:24,783 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:24,783 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. after waiting 0 ms 2024-11-24T17:14:24,783 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:24,783 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:24,783 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 93d787842f2c5c53adaf00e6e90673e7: Waiting for close lock at 1732468464782Disabling compacts and flushes for region at 1732468464782Disabling writes for close at 1732468464783 (+1 ms)Writing region close event to WAL at 1732468464783Closed at 1732468464783 2024-11-24T17:14:24,787 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T17:14:24,796 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,796 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732468449655.d888fda423b365786e1d269df49d1cfd.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,798 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,798 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,798 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,798 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,799 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,799 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,799 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,799 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,801 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,801 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,801 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,801 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732468449655.71e89563154e9bf392a9193da474463d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,801 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732468449655.67115580ab5444880122d14f983db95c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,802 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,802 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,802 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,802 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732468464788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732468464788"}]},"ts":"1732468464788"} 2024-11-24T17:14:24,904 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-24T17:14:24,928 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T17:14:24,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732468464929"}]},"ts":"1732468464929"} 2024-11-24T17:14:24,948 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-24T17:14:24,949 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {a23eb99f71f0=0} racks are {/default-rack=0} 2024-11-24T17:14:25,064 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T17:14:25,065 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T17:14:25,065 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T17:14:25,065 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T17:14:25,065 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T17:14:25,065 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T17:14:25,065 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T17:14:25,065 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T17:14:25,065 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T17:14:25,065 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T17:14:25,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d888fda423b365786e1d269df49d1cfd, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=94439407a643ae820e9015188cf23ed6, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=448f6154a795fbeb01b36798bcbacb8f, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4707e344fe9a8cd93af441dfe2b9492e, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89f671091abaef7d04c744da188c0bf9, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3132e986d91cd07c924412b1164b0ada, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acf1021fd9104e08951bd81c1946b074, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a61743f7cd3883457b967d55f98541e1, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b1ebd571dd5d321046e5067f5b19ad97, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=68cc51974c64ffdbd8bb6b53a87450da, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3bf176f82dd506ad26a9f1d6ebd6f33a, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ba4002d0b6663684be2c126a81b2c74, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4ed93b4907aeb35b406e6de08e103616, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=71e89563154e9bf392a9193da474463d, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=67115580ab5444880122d14f983db95c, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=61f230c444c764032b318ed0a81000ab, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f35eb03f4e248bda613971e436befd3, ASSIGN}] 2024-11-24T17:14:25,103 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f35eb03f4e248bda613971e436befd3, ASSIGN 2024-11-24T17:14:25,110 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4f35eb03f4e248bda613971e436befd3, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,117 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=61f230c444c764032b318ed0a81000ab, ASSIGN 2024-11-24T17:14:25,123 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=61f230c444c764032b318ed0a81000ab, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:25,132 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:25,137 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,145 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=67115580ab5444880122d14f983db95c, ASSIGN 2024-11-24T17:14:25,157 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=67115580ab5444880122d14f983db95c, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,167 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=71e89563154e9bf392a9193da474463d, ASSIGN 2024-11-24T17:14:25,174 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=71e89563154e9bf392a9193da474463d, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:25,179 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:25,183 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,186 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:25,191 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,194 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4ed93b4907aeb35b406e6de08e103616, ASSIGN 2024-11-24T17:14:25,198 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4ed93b4907aeb35b406e6de08e103616, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,205 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7ba4002d0b6663684be2c126a81b2c74, ASSIGN 2024-11-24T17:14:25,210 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7ba4002d0b6663684be2c126a81b2c74, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:25,215 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, ASSIGN 2024-11-24T17:14:25,217 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,220 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3bf176f82dd506ad26a9f1d6ebd6f33a, ASSIGN 2024-11-24T17:14:25,222 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3bf176f82dd506ad26a9f1d6ebd6f33a, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,224 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=68cc51974c64ffdbd8bb6b53a87450da, ASSIGN 2024-11-24T17:14:25,228 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=68cc51974c64ffdbd8bb6b53a87450da, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:25,233 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b1ebd571dd5d321046e5067f5b19ad97, ASSIGN 2024-11-24T17:14:25,238 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b1ebd571dd5d321046e5067f5b19ad97, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,242 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:25,245 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,249 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a61743f7cd3883457b967d55f98541e1, ASSIGN 2024-11-24T17:14:25,252 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a61743f7cd3883457b967d55f98541e1, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,256 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acf1021fd9104e08951bd81c1946b074, ASSIGN 2024-11-24T17:14:25,261 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=acf1021fd9104e08951bd81c1946b074, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:25,264 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3132e986d91cd07c924412b1164b0ada, ASSIGN 2024-11-24T17:14:25,268 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3132e986d91cd07c924412b1164b0ada, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:25,268 INFO [a23eb99f71f0:40213 {}] balancer.BaseLoadBalancer(388): Reassigned 16 regions. 16 retained the pre-restart assignment. 2024-11-24T17:14:25,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,275 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,276 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,279 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=4f35eb03f4e248bda613971e436befd3, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,279 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=67115580ab5444880122d14f983db95c, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,280 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4ed93b4907aeb35b406e6de08e103616, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,280 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=a61743f7cd3883457b967d55f98541e1, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=61f230c444c764032b318ed0a81000ab, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=71e89563154e9bf392a9193da474463d, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,283 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=7ba4002d0b6663684be2c126a81b2c74, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,285 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=acf1021fd9104e08951bd81c1946b074, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:25,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN because future has completed 2024-11-24T17:14:25,290 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,294 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89f671091abaef7d04c744da188c0bf9, ASSIGN 2024-11-24T17:14:25,298 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=89f671091abaef7d04c744da188c0bf9, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN because future has completed 2024-11-24T17:14:25,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:25,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN because future has completed 2024-11-24T17:14:25,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:25,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, ASSIGN because future has completed 2024-11-24T17:14:25,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN because future has completed 2024-11-24T17:14:25,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4f35eb03f4e248bda613971e436befd3, ASSIGN because future has completed 2024-11-24T17:14:25,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=67115580ab5444880122d14f983db95c, ASSIGN because future has completed 2024-11-24T17:14:25,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f35eb03f4e248bda613971e436befd3, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4ed93b4907aeb35b406e6de08e103616, ASSIGN because future has completed 2024-11-24T17:14:25,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3bf176f82dd506ad26a9f1d6ebd6f33a, ASSIGN because future has completed 2024-11-24T17:14:25,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bf176f82dd506ad26a9f1d6ebd6f33a, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b1ebd571dd5d321046e5067f5b19ad97, ASSIGN because future has completed 2024-11-24T17:14:25,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a61743f7cd3883457b967d55f98541e1, ASSIGN because future has completed 2024-11-24T17:14:25,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=61f230c444c764032b318ed0a81000ab, ASSIGN because future has completed 2024-11-24T17:14:25,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=71e89563154e9bf392a9193da474463d, ASSIGN because future has completed 2024-11-24T17:14:25,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71e89563154e9bf392a9193da474463d, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:25,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7ba4002d0b6663684be2c126a81b2c74, ASSIGN because future has completed 2024-11-24T17:14:25,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=68cc51974c64ffdbd8bb6b53a87450da, ASSIGN because future has completed 2024-11-24T17:14:25,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=acf1021fd9104e08951bd81c1946b074, ASSIGN because future has completed 2024-11-24T17:14:25,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure acf1021fd9104e08951bd81c1946b074, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:25,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68cc51974c64ffdbd8bb6b53a87450da, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:25,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ba4002d0b6663684be2c126a81b2c74, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:25,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61f230c444c764032b318ed0a81000ab, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:25,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure a61743f7cd3883457b967d55f98541e1, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1ebd571dd5d321046e5067f5b19ad97, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ed93b4907aeb35b406e6de08e103616, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,420 INFO [a23eb99f71f0:40213 {}] balancer.BaseLoadBalancer(388): Reassigned 3 regions. 3 retained the pre-restart assignment. 2024-11-24T17:14:25,426 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,428 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=89f671091abaef7d04c744da188c0bf9, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,429 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=3132e986d91cd07c924412b1164b0ada, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67115580ab5444880122d14f983db95c, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN because future has completed 2024-11-24T17:14:25,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=89f671091abaef7d04c744da188c0bf9, ASSIGN because future has completed 2024-11-24T17:14:25,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3132e986d91cd07c924412b1164b0ada, ASSIGN because future has completed 2024-11-24T17:14:25,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3132e986d91cd07c924412b1164b0ada, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:25,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89f671091abaef7d04c744da188c0bf9, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:25,472 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:25,473 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 58c2a35cef97d825f565d67ca7006c36, NAME => 'test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-24T17:14:25,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12eb6cfa7992f8b2d57701a44d5e5526, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:25,474 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,474 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,474 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,474 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:25,480 INFO [StoreOpener-58c2a35cef97d825f565d67ca7006c36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,485 INFO [StoreOpener-58c2a35cef97d825f565d67ca7006c36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c2a35cef97d825f565d67ca7006c36 columnFamilyName family 2024-11-24T17:14:25,485 DEBUG [StoreOpener-58c2a35cef97d825f565d67ca7006c36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,487 INFO [StoreOpener-58c2a35cef97d825f565d67ca7006c36-1 {}] regionserver.HStore(327): Store=58c2a35cef97d825f565d67ca7006c36/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,487 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,489 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,490 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:25,490 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,490 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,495 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,496 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4707e344fe9a8cd93af441dfe2b9492e, ASSIGN 2024-11-24T17:14:25,499 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4707e344fe9a8cd93af441dfe2b9492e, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:25,504 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=448f6154a795fbeb01b36798bcbacb8f, ASSIGN 2024-11-24T17:14:25,505 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T17:14:25,507 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T17:14:25,507 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=448f6154a795fbeb01b36798bcbacb8f, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59243, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T17:14:25,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46087, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T17:14:25,514 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:25,516 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,519 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:25,521 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:25,522 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 89f671091abaef7d04c744da188c0bf9, NAME => 'test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-24T17:14:25,522 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/58c2a35cef97d825f565d67ca7006c36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,523 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,523 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,524 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,524 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 58c2a35cef97d825f565d67ca7006c36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60507848, jitterRate=-0.09836280345916748}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,524 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,524 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:25,524 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,525 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 58c2a35cef97d825f565d67ca7006c36: Running coprocessor pre-open hook at 1732468465474Writing region info on filesystem at 1732468465474Initializing all the Stores at 1732468465476 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465477 (+1 ms)Cleaning up temporary data from old regions at 1732468465490 (+13 ms)Running coprocessor post-open hooks at 1732468465524 (+34 ms)Region opened successfully at 1732468465525 (+1 ms) 2024-11-24T17:14:25,527 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:25,528 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 61f230c444c764032b318ed0a81000ab, NAME => 'test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-24T17:14:25,528 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,528 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,529 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,529 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,530 INFO [StoreOpener-89f671091abaef7d04c744da188c0bf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,530 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:25,530 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36., pid=31, masterSystemTime=1732468465460 2024-11-24T17:14:25,534 INFO [StoreOpener-89f671091abaef7d04c744da188c0bf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f671091abaef7d04c744da188c0bf9 columnFamilyName family 2024-11-24T17:14:25,534 DEBUG [StoreOpener-89f671091abaef7d04c744da188c0bf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,537 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN; state=OFFLINE, location=a23eb99f71f0,44781,1732468442596; forceNewPlan=false, retain=false 2024-11-24T17:14:25,540 INFO [StoreOpener-61f230c444c764032b318ed0a81000ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,542 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:25,542 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:25,543 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:25,543 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f16dc15c0e0aff7efc028181a4fdbe3, NAME => 'test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-24T17:14:25,544 INFO [StoreOpener-61f230c444c764032b318ed0a81000ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61f230c444c764032b318ed0a81000ab columnFamilyName family 2024-11-24T17:14:25,544 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,544 DEBUG [StoreOpener-61f230c444c764032b318ed0a81000ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,544 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,544 INFO [StoreOpener-89f671091abaef7d04c744da188c0bf9-1 {}] regionserver.HStore(327): Store=89f671091abaef7d04c744da188c0bf9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,544 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,544 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,544 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,545 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=58c2a35cef97d825f565d67ca7006c36, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,546 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,547 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,548 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,548 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,549 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=94439407a643ae820e9015188cf23ed6, ASSIGN 2024-11-24T17:14:25,552 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=94439407a643ae820e9015188cf23ed6, ASSIGN; state=OFFLINE, location=a23eb99f71f0,38279,1732468442718; forceNewPlan=false, retain=false 2024-11-24T17:14:25,552 INFO [StoreOpener-61f230c444c764032b318ed0a81000ab-1 {}] regionserver.HStore(327): Store=61f230c444c764032b318ed0a81000ab/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,552 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,553 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,555 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:25,556 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,557 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,557 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,561 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,574 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/89f671091abaef7d04c744da188c0bf9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-24T17:14:25,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,44781,1732468442596 in 266 msec 2024-11-24T17:14:25,576 INFO [StoreOpener-6f16dc15c0e0aff7efc028181a4fdbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,576 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 89f671091abaef7d04c744da188c0bf9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73276322, jitterRate=0.09190228581428528}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,576 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:25,577 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 89f671091abaef7d04c744da188c0bf9: Running coprocessor pre-open hook at 1732468465525Writing region info on filesystem at 1732468465525Initializing all the Stores at 1732468465528 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465528Cleaning up temporary data from old regions at 1732468465548 (+20 ms)Running coprocessor post-open hooks at 1732468465576 (+28 ms)Region opened successfully at 1732468465577 (+1 ms) 2024-11-24T17:14:25,579 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/61f230c444c764032b318ed0a81000ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,580 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9., pid=45, masterSystemTime=1732468465504 2024-11-24T17:14:25,580 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 61f230c444c764032b318ed0a81000ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63532988, jitterRate=-0.05328470468521118}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,581 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:25,581 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 61f230c444c764032b318ed0a81000ab: Running coprocessor pre-open hook at 1732468465529Writing region info on filesystem at 1732468465529Initializing all the Stores at 1732468465532 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465532Cleaning up temporary data from old regions at 1732468465557 (+25 ms)Running coprocessor post-open hooks at 1732468465581 (+24 ms)Region opened successfully at 1732468465581 2024-11-24T17:14:25,581 INFO [StoreOpener-6f16dc15c0e0aff7efc028181a4fdbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f16dc15c0e0aff7efc028181a4fdbe3 columnFamilyName family 2024-11-24T17:14:25,582 DEBUG [StoreOpener-6f16dc15c0e0aff7efc028181a4fdbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,583 INFO [StoreOpener-6f16dc15c0e0aff7efc028181a4fdbe3-1 {}] regionserver.HStore(327): Store=6f16dc15c0e0aff7efc028181a4fdbe3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,583 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab., pid=39, masterSystemTime=1732468465506 2024-11-24T17:14:25,583 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,585 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN in 508 msec 2024-11-24T17:14:25,587 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,588 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,588 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,588 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:25,588 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:25,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=61f230c444c764032b318ed0a81000ab, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,588 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:25,589 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 71e89563154e9bf392a9193da474463d, NAME => 'test,vvv,1732468449655.71e89563154e9bf392a9193da474463d.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-24T17:14:25,589 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,589 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,vvv,1732468449655.71e89563154e9bf392a9193da474463d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,589 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,589 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,592 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:25,593 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:25,593 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:25,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61f230c444c764032b318ed0a81000ab, server=a23eb99f71f0,34777,1732468442786 because future has completed 2024-11-24T17:14:25,593 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ed93b4907aeb35b406e6de08e103616, NAME => 'test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-24T17:14:25,593 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,594 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,594 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,594 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,594 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,596 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,596 INFO [StoreOpener-71e89563154e9bf392a9193da474463d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89f671091abaef7d04c744da188c0bf9, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:25,601 INFO [StoreOpener-71e89563154e9bf392a9193da474463d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 71e89563154e9bf392a9193da474463d columnFamilyName family 2024-11-24T17:14:25,601 DEBUG [StoreOpener-71e89563154e9bf392a9193da474463d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,602 INFO [StoreOpener-71e89563154e9bf392a9193da474463d-1 {}] regionserver.HStore(327): Store=71e89563154e9bf392a9193da474463d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,602 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,603 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,604 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,605 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,605 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=29 2024-11-24T17:14:25,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 61f230c444c764032b318ed0a81000ab, server=a23eb99f71f0,34777,1732468442786 in 212 msec 2024-11-24T17:14:25,609 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,609 INFO [StoreOpener-4ed93b4907aeb35b406e6de08e103616-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,609 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/6f16dc15c0e0aff7efc028181a4fdbe3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,610 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 6f16dc15c0e0aff7efc028181a4fdbe3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73182167, jitterRate=0.09049926698207855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,611 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:25,611 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 6f16dc15c0e0aff7efc028181a4fdbe3: Running coprocessor pre-open hook at 1732468465545Writing region info on filesystem at 1732468465545Initializing all the Stores at 1732468465547 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465548 (+1 ms)Cleaning up temporary data from old regions at 1732468465588 (+40 ms)Running coprocessor post-open hooks at 1732468465611 (+23 ms)Region opened successfully at 1732468465611 2024-11-24T17:14:25,612 INFO [StoreOpener-4ed93b4907aeb35b406e6de08e103616-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed93b4907aeb35b406e6de08e103616 columnFamilyName family 2024-11-24T17:14:25,612 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3., pid=32, masterSystemTime=1732468465460 2024-11-24T17:14:25,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=61f230c444c764032b318ed0a81000ab, ASSIGN in 538 msec 2024-11-24T17:14:25,612 DEBUG [StoreOpener-4ed93b4907aeb35b406e6de08e103616-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,614 INFO [StoreOpener-4ed93b4907aeb35b406e6de08e103616-1 {}] regionserver.HStore(327): Store=4ed93b4907aeb35b406e6de08e103616/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,615 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/71e89563154e9bf392a9193da474463d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,615 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,615 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:25,616 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:25,616 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 71e89563154e9bf392a9193da474463d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66042176, jitterRate=-0.01589488983154297}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,616 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:25,616 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 71e89563154e9bf392a9193da474463d: Running coprocessor pre-open hook at 1732468465590Writing region info on filesystem at 1732468465590Initializing all the Stores at 1732468465592 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465592Cleaning up temporary data from old regions at 1732468465605 (+13 ms)Running coprocessor post-open hooks at 1732468465616 (+11 ms)Region opened successfully at 1732468465616 2024-11-24T17:14:25,616 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,617 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732468449655.71e89563154e9bf392a9193da474463d., pid=35, masterSystemTime=1732468465506 2024-11-24T17:14:25,622 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,623 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:25,623 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:25,623 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:25,623 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,623 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,623 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => acf1021fd9104e08951bd81c1946b074, NAME => 'test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-24T17:14:25,624 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,624 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,624 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,624 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,628 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:25,628 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 93d787842f2c5c53adaf00e6e90673e7, NAME => 'test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-24T17:14:25,629 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,629 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,629 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,629 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=12 2024-11-24T17:14:25,629 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 89f671091abaef7d04c744da188c0bf9, server=a23eb99f71f0,38279,1732468442718 in 164 msec 2024-11-24T17:14:25,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=89f671091abaef7d04c744da188c0bf9, ASSIGN in 562 msec 2024-11-24T17:14:25,636 INFO [StoreOpener-acf1021fd9104e08951bd81c1946b074-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,638 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=71e89563154e9bf392a9193da474463d, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,640 INFO [StoreOpener-acf1021fd9104e08951bd81c1946b074-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acf1021fd9104e08951bd81c1946b074 columnFamilyName family 2024-11-24T17:14:25,640 DEBUG [StoreOpener-acf1021fd9104e08951bd81c1946b074-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,645 INFO [StoreOpener-acf1021fd9104e08951bd81c1946b074-1 {}] regionserver.HStore(327): Store=acf1021fd9104e08951bd81c1946b074/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,645 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,647 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d888fda423b365786e1d269df49d1cfd, ASSIGN 2024-11-24T17:14:25,647 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71e89563154e9bf392a9193da474463d, server=a23eb99f71f0,34777,1732468442786 because future has completed 2024-11-24T17:14:25,648 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,649 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,649 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,650 INFO [a23eb99f71f0:40213 {}] balancer.BaseLoadBalancer(388): Reassigned 6 regions. 6 retained the pre-restart assignment. 2024-11-24T17:14:25,650 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=71e89563154e9bf392a9193da474463d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:25,650 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d888fda423b365786e1d269df49d1cfd, ASSIGN; state=OFFLINE, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:25,653 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,653 INFO [StoreOpener-93d787842f2c5c53adaf00e6e90673e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,654 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=448f6154a795fbeb01b36798bcbacb8f, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,656 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=94439407a643ae820e9015188cf23ed6, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,657 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:25,665 INFO [StoreOpener-93d787842f2c5c53adaf00e6e90673e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93d787842f2c5c53adaf00e6e90673e7 columnFamilyName family 2024-11-24T17:14:25,665 DEBUG [StoreOpener-93d787842f2c5c53adaf00e6e90673e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,667 INFO [StoreOpener-93d787842f2c5c53adaf00e6e90673e7-1 {}] regionserver.HStore(327): Store=93d787842f2c5c53adaf00e6e90673e7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,667 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,668 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4ed93b4907aeb35b406e6de08e103616/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,669 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,669 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 4ed93b4907aeb35b406e6de08e103616; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60067242, jitterRate=-0.10492834448814392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,669 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:25,669 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 4ed93b4907aeb35b406e6de08e103616: Running coprocessor pre-open hook at 1732468465594Writing region info on filesystem at 1732468465594Initializing all the Stores at 1732468465595 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465596 (+1 ms)Cleaning up temporary data from old regions at 1732468465623 (+27 ms)Running coprocessor post-open hooks at 1732468465669 (+46 ms)Region opened successfully at 1732468465669 2024-11-24T17:14:25,669 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN because future has completed 2024-11-24T17:14:25,672 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,672 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616., pid=42, masterSystemTime=1732468465504 2024-11-24T17:14:25,672 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=26 2024-11-24T17:14:25,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 71e89563154e9bf392a9193da474463d, server=a23eb99f71f0,34777,1732468442786 in 311 msec 2024-11-24T17:14:25,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN because future has completed 2024-11-24T17:14:25,676 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:25,676 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:25,676 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/acf1021fd9104e08951bd81c1946b074/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,676 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:25,677 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f35eb03f4e248bda613971e436befd3, NAME => 'test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-24T17:14:25,677 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,677 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,677 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened acf1021fd9104e08951bd81c1946b074; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63204148, jitterRate=-0.058184802532196045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,677 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,677 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:25,677 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,677 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for acf1021fd9104e08951bd81c1946b074: Running coprocessor pre-open hook at 1732468465624Writing region info on filesystem at 1732468465624Initializing all the Stores at 1732468465627 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465627Cleaning up temporary data from old regions at 1732468465649 (+22 ms)Running coprocessor post-open hooks at 1732468465677 (+28 ms)Region opened successfully at 1732468465677 2024-11-24T17:14:25,677 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN because future has completed 2024-11-24T17:14:25,681 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074., pid=36, masterSystemTime=1732468465506 2024-11-24T17:14:25,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=71e89563154e9bf392a9193da474463d, ASSIGN in 606 msec 2024-11-24T17:14:25,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:25,685 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:25,685 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:25,685 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:25,686 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 3132e986d91cd07c924412b1164b0ada, NAME => 'test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-24T17:14:25,686 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,686 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,686 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,687 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=448f6154a795fbeb01b36798bcbacb8f, ASSIGN because future has completed 2024-11-24T17:14:25,688 INFO [StoreOpener-4f35eb03f4e248bda613971e436befd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=94439407a643ae820e9015188cf23ed6, ASSIGN because future has completed 2024-11-24T17:14:25,692 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/93d787842f2c5c53adaf00e6e90673e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94439407a643ae820e9015188cf23ed6, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,699 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 93d787842f2c5c53adaf00e6e90673e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63056233, jitterRate=-0.06038890779018402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,700 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:25,700 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 93d787842f2c5c53adaf00e6e90673e7: Running coprocessor pre-open hook at 1732468465629Writing region info on filesystem at 1732468465629Initializing all the Stores at 1732468465631 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465631Cleaning up temporary data from old regions at 1732468465672 (+41 ms)Running coprocessor post-open hooks at 1732468465700 (+28 ms)Region opened successfully at 1732468465700 2024-11-24T17:14:25,701 INFO [StoreOpener-4f35eb03f4e248bda613971e436befd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f35eb03f4e248bda613971e436befd3 columnFamilyName family 2024-11-24T17:14:25,701 DEBUG [StoreOpener-4f35eb03f4e248bda613971e436befd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,702 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7., pid=49, masterSystemTime=1732468465619 2024-11-24T17:14:25,702 INFO [StoreOpener-3132e986d91cd07c924412b1164b0ada-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,705 INFO [StoreOpener-3132e986d91cd07c924412b1164b0ada-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3132e986d91cd07c924412b1164b0ada columnFamilyName family 2024-11-24T17:14:25,705 DEBUG [StoreOpener-3132e986d91cd07c924412b1164b0ada-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,706 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:25,706 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:25,706 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:25,706 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => a6d2969d9f925c5b8cb0ff244090dcd4, NAME => 'test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-24T17:14:25,706 INFO [StoreOpener-3132e986d91cd07c924412b1164b0ada-1 {}] regionserver.HStore(327): Store=3132e986d91cd07c924412b1164b0ada/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,706 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,707 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,707 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,707 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,707 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4707e344fe9a8cd93af441dfe2b9492e, ASSIGN because future has completed 2024-11-24T17:14:25,708 INFO [StoreOpener-4f35eb03f4e248bda613971e436befd3-1 {}] regionserver.HStore(327): Store=4f35eb03f4e248bda613971e436befd3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,708 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4707e344fe9a8cd93af441dfe2b9492e, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:25,720 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,721 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,721 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,722 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,722 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=93d787842f2c5c53adaf00e6e90673e7, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:25,722 INFO [StoreOpener-a6d2969d9f925c5b8cb0ff244090dcd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,724 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,725 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,725 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,725 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 448f6154a795fbeb01b36798bcbacb8f, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:25,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:25,736 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,737 INFO [StoreOpener-a6d2969d9f925c5b8cb0ff244090dcd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6d2969d9f925c5b8cb0ff244090dcd4 columnFamilyName family 2024-11-24T17:14:25,737 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,737 DEBUG [StoreOpener-a6d2969d9f925c5b8cb0ff244090dcd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:25,737 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:25,740 INFO [StoreOpener-a6d2969d9f925c5b8cb0ff244090dcd4-1 {}] regionserver.HStore(327): Store=a6d2969d9f925c5b8cb0ff244090dcd4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:25,740 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=25 2024-11-24T17:14:25,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,44781,1732468442596 in 244 msec 2024-11-24T17:14:25,742 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,743 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,744 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,744 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN in 673 msec 2024-11-24T17:14:25,746 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3132e986d91cd07c924412b1164b0ada/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,747 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 3132e986d91cd07c924412b1164b0ada; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68959330, jitterRate=0.027574092149734497}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,747 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:25,747 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 3132e986d91cd07c924412b1164b0ada: Running coprocessor pre-open hook at 1732468465687Writing region info on filesystem at 1732468465687Initializing all the Stores at 1732468465690 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465690Cleaning up temporary data from old regions at 1732468465725 (+35 ms)Running coprocessor post-open hooks at 1732468465747 (+22 ms)Region opened successfully at 1732468465747 2024-11-24T17:14:25,748 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,754 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,755 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:25,756 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada., pid=44, masterSystemTime=1732468465506 2024-11-24T17:14:25,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure acf1021fd9104e08951bd81c1946b074, server=a23eb99f71f0,34777,1732468442786 because future has completed 2024-11-24T17:14:25,761 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:25,761 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:25,761 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:25,761 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:25,761 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 68cc51974c64ffdbd8bb6b53a87450da, NAME => 'test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-24T17:14:25,762 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:25,762 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:25,762 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:25,762 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:25,764 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4f35eb03f4e248bda613971e436befd3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ed93b4907aeb35b406e6de08e103616, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:25,766 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:25,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3132e986d91cd07c924412b1164b0ada, server=a23eb99f71f0,34777,1732468442786 because future has completed 2024-11-24T17:14:25,772 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 4f35eb03f4e248bda613971e436befd3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70943547, jitterRate=0.05714122951030731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,772 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:25,772 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 4f35eb03f4e248bda613971e436befd3: Running coprocessor pre-open hook at 1732468465678Writing region info on filesystem at 1732468465678Initializing all the Stores at 1732468465680 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465680Cleaning up temporary data from old regions at 1732468465725 (+45 ms)Running coprocessor post-open hooks at 1732468465772 (+47 ms)Region opened successfully at 1732468465772 2024-11-24T17:14:25,774 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a6d2969d9f925c5b8cb0ff244090dcd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:25,774 INFO [StoreOpener-68cc51974c64ffdbd8bb6b53a87450da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:25,775 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened a6d2969d9f925c5b8cb0ff244090dcd4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67017234, jitterRate=-0.0013653934001922607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:25,776 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:25,776 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for a6d2969d9f925c5b8cb0ff244090dcd4: Running coprocessor pre-open hook at 1732468465707Writing region info on filesystem at 1732468465707Initializing all the Stores at 1732468465721 (+14 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465721Cleaning up temporary data from old regions at 1732468465744 (+23 ms)Running coprocessor post-open hooks at 1732468465776 (+32 ms)Region opened successfully at 1732468465776 2024-11-24T17:14:25,801 INFO [a23eb99f71f0:40213 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T17:14:25,840 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:25,859 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:25,859 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:25,948 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:25,963 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:25,964 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,051 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,066 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,066 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,154 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,168 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,168 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,261 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,271 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,271 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,365 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,375 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,375 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,470 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,478 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,478 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,574 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,580 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,580 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,678 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,683 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,683 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,781 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,785 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,785 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,883 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,887 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,888 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,985 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,989 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:26,989 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,087 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,091 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,091 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,189 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,193 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,193 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,291 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,295 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,295 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,393 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,397 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,397 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,495 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,499 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,499 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,598 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,601 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,602 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,700 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,703 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,703 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,803 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,806 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,806 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,905 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,908 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:27,908 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:28,007 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,009 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,009 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,108 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,110 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,110 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,211 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,213 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,213 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,313 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,315 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,315 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,416 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,423 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,423 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,524 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,527 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,527 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,630 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,631 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,631 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,734 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,734 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,735 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,837 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,838 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,838 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,939 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,939 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:28,940 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,041 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,041 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,042 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,143 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,143 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,144 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,245 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,246 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,246 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,348 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,348 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,349 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,451 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,451 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,452 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,554 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,554 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,555 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,657 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,657 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,658 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,759 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,760 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,760 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,861 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,861 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,862 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,963 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,963 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:29,964 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,066 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,066 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,067 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,169 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,169 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,170 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,233 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-24T17:14:30,239 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T17:14:30,271 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,271 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,272 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,374 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,375 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,375 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,477 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,477 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,481 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,581 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,582 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,595 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,696 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,696 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,701 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,801 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,801 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,804 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,904 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,905 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:30,906 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,007 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,007 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,008 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,011 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:14:31,109 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,109 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,110 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,211 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,211 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,212 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,313 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,313 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,313 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,414 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,415 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,415 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,516 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,517 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,517 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,618 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,619 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,619 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-24T17:14:31,679 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-24T17:14:31,720 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,721 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,721 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,822 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,822 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,822 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,924 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,924 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:31,924 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,026 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,027 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,027 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,129 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,129 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,129 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,231 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,231 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,231 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,332 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,333 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,333 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,435 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,435 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,436 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,538 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,538 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,539 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,640 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,641 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,641 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,743 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,744 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,744 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,847 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,848 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,848 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,950 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,950 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:32,951 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,052 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,052 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,052 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,154 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,154 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,154 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,256 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,256 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,256 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,358 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,358 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,359 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,461 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,462 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,462 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,564 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,564 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,564 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,666 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,666 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,666 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,767 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,768 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,768 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,869 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,870 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,870 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,971 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,973 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:33,973 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,074 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,075 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,075 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,176 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,177 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,177 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,278 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,278 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,278 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,380 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,381 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,381 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,482 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,484 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,485 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,586 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,586 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,587 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,688 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,688 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,688 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,790 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,790 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,790 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,892 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,893 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,894 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,996 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,996 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:34,997 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,098 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,099 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,099 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,202 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,204 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,204 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,306 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,307 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,307 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,409 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,410 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,410 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,511 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,512 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,512 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,613 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,613 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,613 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,716 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,717 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,717 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,818 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,818 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,819 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,920 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,920 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:35,920 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,021 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,022 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,022 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,123 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,123 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,123 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,225 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,226 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,226 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,327 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,328 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,328 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,431 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,432 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,432 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,535 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,536 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,536 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,638 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,640 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,640 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,741 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,751 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,751 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,836 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1903757570_22 at /127.0.0.1:56814 [Receiving block BP-1082952430-172.17.0.3-1732468435659:blk_1073741835_1011] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 11059ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data3/, blockId=1073741835, seqno=4 2024-11-24T17:14:36,837 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_108502128_22 at /127.0.0.1:53534 [Receiving block BP-1082952430-172.17.0.3-1732468435659:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 11060ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data2/, blockId=1073741830, seqno=235 2024-11-24T17:14:36,839 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1903757570_22 at /127.0.0.1:53574 [Receiving block BP-1082952430-172.17.0.3-1732468435659:blk_1073741835_1011] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 11059ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data2/, blockId=1073741835, seqno=4 2024-11-24T17:14:36,836 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1903757570_22 at /127.0.0.1:42178 [Receiving block BP-1082952430-172.17.0.3-1732468435659:blk_1073741835_1011] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 11059ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data5/, blockId=1073741835, seqno=4 2024-11-24T17:14:36,839 INFO [StoreOpener-68cc51974c64ffdbd8bb6b53a87450da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68cc51974c64ffdbd8bb6b53a87450da columnFamilyName family 2024-11-24T17:14:36,836 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_995739179_22 at /127.0.0.1:56806 [Receiving block BP-1082952430-172.17.0.3-1732468435659:blk_1073741834_1010] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 11055ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c70f698-b7f9-3115-b4be-62446e947b38/cluster_644f43a8-c061-bd47-7e28-d1959be30fbc/data/data4/, blockId=1073741834, seqno=3 2024-11-24T17:14:36,841 DEBUG [StoreOpener-68cc51974c64ffdbd8bb6b53a87450da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:36,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] monitor.StreamSlowMonitor(154): Slow datanode: DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK], data length=377, duration=11068ms, unfinishedReplicas=0, lastAckTimestamp=1732468465775, monitor name: defaultMonitorName 2024-11-24T17:14:36,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] monitor.StreamSlowMonitor(154): Slow datanode: DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK], data length=886, duration=11065ms, unfinishedReplicas=0, lastAckTimestamp=1732468465777, monitor name: defaultMonitorName 2024-11-24T17:14:36,842 INFO [AsyncFSWAL-0-hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5-prefix:a23eb99f71f0,38279,1732468442718 {}] wal.AbstractFSWAL(1368): Slow sync cost: 11069 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK], DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK], DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK]] 2024-11-24T17:14:36,844 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3., pid=33, masterSystemTime=1732468465504 2024-11-24T17:14:36,846 INFO [AsyncFSWAL-0-hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData-prefix:a23eb99f71f0,40213,1732468441272 {}] wal.AbstractFSWAL(1368): Slow sync cost: 11069 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK], DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK]] 2024-11-24T17:14:36,846 INFO [AsyncFSWAL-0-hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5-prefix:a23eb99f71f0,44781,1732468442596 {}] wal.AbstractFSWAL(1368): Slow sync cost: 11069 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK], DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK], DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK]] 2024-11-24T17:14:36,847 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4., pid=48, masterSystemTime=1732468465619 2024-11-24T17:14:36,849 WARN [AsyncFSWAL-0-hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5-prefix:a23eb99f71f0,44781,1732468442596 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=11069 ms, threshold=10000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK], DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK], DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK]] 2024-11-24T17:14:36,849 WARN [AsyncFSWAL-0-hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5-prefix:a23eb99f71f0,38279,1732468442718 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=11069 ms, threshold=10000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK], DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK], DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK]] 2024-11-24T17:14:36,851 WARN [AsyncFSWAL-0-hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData-prefix:a23eb99f71f0,40213,1732468441272 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=11069 ms, threshold=10000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK], DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK]] 2024-11-24T17:14:36,851 INFO [AsyncFSWAL-0-hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData-prefix:a23eb99f71f0,40213,1732468441272 {}] wal.AbstractFSWAL(1368): Slow sync cost: 1386 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK], DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK]] 2024-11-24T17:14:36,851 DEBUG [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a23eb99f71f0%2C44781%2C1732468442596:(num 1732468446752) roll requested 2024-11-24T17:14:36,854 DEBUG [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a23eb99f71f0%2C38279%2C1732468442718:(num 1732468446724) roll requested 2024-11-24T17:14:36,856 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a23eb99f71f0%2C40213%2C1732468441272:(num 1732468444106) roll requested 2024-11-24T17:14:36,859 INFO [StoreOpener-68cc51974c64ffdbd8bb6b53a87450da-1 {}] regionserver.HStore(327): Store=68cc51974c64ffdbd8bb6b53a87450da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:36,859 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:36,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=23 2024-11-24T17:14:36,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 4ed93b4907aeb35b406e6de08e103616, server=a23eb99f71f0,38279,1732468442718 in 356 msec 2024-11-24T17:14:36,867 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,867 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,867 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,868 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:36,868 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:36,868 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:36,868 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:36,868 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 3963c202e600d921e58afb7a1966f4f0, NAME => 'test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-24T17:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,871 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:36,871 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:36,871 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:36,871 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => a61743f7cd3883457b967d55f98541e1, NAME => 'test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-24T17:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:36,878 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:36,882 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:36,882 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:36,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f35eb03f4e248bda613971e436befd3, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:36,885 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:36,887 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:36,893 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=48, ppid=21, state=RUNNABLE, hasLock=true; OpenRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:36,893 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,38279,1732468442718, table=test, region=4f35eb03f4e248bda613971e436befd3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,894 DEBUG [regionserver/a23eb99f71f0:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.1732468476851, exclude list is [], retry=0 2024-11-24T17:14:36,896 DEBUG [regionserver/a23eb99f71f0:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718/a23eb99f71f0%2C38279%2C1732468442718.1732468476854, exclude list is [], retry=0 2024-11-24T17:14:36,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK] 2024-11-24T17:14:36,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:36,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:36,915 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/WALs/a23eb99f71f0,40213,1732468441272/a23eb99f71f0%2C40213%2C1732468441272.1732468476856, exclude list is [], retry=0 2024-11-24T17:14:36,916 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/68cc51974c64ffdbd8bb6b53a87450da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:36,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:36,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:36,918 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 68cc51974c64ffdbd8bb6b53a87450da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59905994, jitterRate=-0.10733112692832947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:36,918 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:36,918 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 68cc51974c64ffdbd8bb6b53a87450da: Running coprocessor pre-open hook at 1732468465762Writing region info on filesystem at 1732468465762Initializing all the Stores at 1732468465771 (+9 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468465771Cleaning up temporary data from old regions at 1732468476882 (+11111 ms)Running coprocessor post-open hooks at 1732468476918 (+36 ms)Region opened successfully at 1732468476918 2024-11-24T17:14:36,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK] 2024-11-24T17:14:36,923 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da., pid=37, masterSystemTime=1732468465506 2024-11-24T17:14:36,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=21 2024-11-24T17:14:36,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,44781,1732468442596 in 11.4130 sec 2024-11-24T17:14:36,928 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:36,928 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:36,928 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:36,929 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ba4002d0b6663684be2c126a81b2c74, NAME => 'test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-24T17:14:36,929 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:36,929 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:36,929 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:36,929 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:36,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, ASSIGN in 11.8570 sec 2024-11-24T17:14:36,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK] 2024-11-24T17:14:36,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:36,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:36,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:36,940 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=37, ppid=19, state=RUNNABLE, hasLock=true; OpenRegionProcedure 68cc51974c64ffdbd8bb6b53a87450da, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:36,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=19 2024-11-24T17:14:36,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 68cc51974c64ffdbd8bb6b53a87450da, server=a23eb99f71f0,34777,1732468442786 in 11.5700 sec 2024-11-24T17:14:36,947 INFO [StoreOpener-3963c202e600d921e58afb7a1966f4f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=68cc51974c64ffdbd8bb6b53a87450da, ASSIGN in 11.8790 sec 2024-11-24T17:14:36,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=30 2024-11-24T17:14:36,960 INFO [StoreOpener-3963c202e600d921e58afb7a1966f4f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3963c202e600d921e58afb7a1966f4f0 columnFamilyName family 2024-11-24T17:14:36,960 DEBUG [StoreOpener-3963c202e600d921e58afb7a1966f4f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:36,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f35eb03f4e248bda613971e436befd3, server=a23eb99f71f0,38279,1732468442718 in 11.6230 sec 2024-11-24T17:14:36,962 INFO [StoreOpener-3963c202e600d921e58afb7a1966f4f0-1 {}] regionserver.HStore(327): Store=3963c202e600d921e58afb7a1966f4f0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:36,962 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,963 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,964 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,965 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,965 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,969 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:36,976 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=3132e986d91cd07c924412b1164b0ada. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,977 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=acf1021fd9104e08951bd81c1946b074. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,981 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:36,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4f35eb03f4e248bda613971e436befd3, ASSIGN in 11.8930 sec 2024-11-24T17:14:36,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4ed93b4907aeb35b406e6de08e103616, ASSIGN in 11.9140 sec 2024-11-24T17:14:36,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d888fda423b365786e1d269df49d1cfd, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:36,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d888fda423b365786e1d269df49d1cfd, ASSIGN because future has completed 2024-11-24T17:14:37,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=14 2024-11-24T17:14:37,006 INFO [StoreOpener-a61743f7cd3883457b967d55f98541e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:37,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 3132e986d91cd07c924412b1164b0ada, server=a23eb99f71f0,34777,1732468442786 in 11.5580 sec 2024-11-24T17:14:37,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3132e986d91cd07c924412b1164b0ada, ASSIGN in 11.9380 sec 2024-11-24T17:14:37,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d888fda423b365786e1d269df49d1cfd, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:37,012 INFO [StoreOpener-7ba4002d0b6663684be2c126a81b2c74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:37,017 INFO [StoreOpener-a61743f7cd3883457b967d55f98541e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a61743f7cd3883457b967d55f98541e1 columnFamilyName family 2024-11-24T17:14:37,017 DEBUG [StoreOpener-a61743f7cd3883457b967d55f98541e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,024 INFO [StoreOpener-a61743f7cd3883457b967d55f98541e1-1 {}] regionserver.HStore(327): Store=a61743f7cd3883457b967d55f98541e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,025 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:37,025 INFO [StoreOpener-7ba4002d0b6663684be2c126a81b2c74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ba4002d0b6663684be2c126a81b2c74 columnFamilyName family 2024-11-24T17:14:37,025 DEBUG [StoreOpener-7ba4002d0b6663684be2c126a81b2c74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,038 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3963c202e600d921e58afb7a1966f4f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,040 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 3963c202e600d921e58afb7a1966f4f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63692812, jitterRate=-0.050903141498565674}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,040 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:37,040 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:37,040 INFO [StoreOpener-7ba4002d0b6663684be2c126a81b2c74-1 {}] regionserver.HStore(327): Store=7ba4002d0b6663684be2c126a81b2c74/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,040 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 3963c202e600d921e58afb7a1966f4f0: Running coprocessor pre-open hook at 1732468476869Writing region info on filesystem at 1732468476869Initializing all the Stores at 1732468476900 (+31 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468476900Cleaning up temporary data from old regions at 1732468476965 (+65 ms)Running coprocessor post-open hooks at 1732468477040 (+75 ms)Region opened successfully at 1732468477040 2024-11-24T17:14:37,041 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:37,042 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:37,042 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:37,042 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:37,043 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:37,044 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:37,045 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:37,045 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:37,046 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:37,050 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:37,084 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,129 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a61743f7cd3883457b967d55f98541e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=15 2024-11-24T17:14:37,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure acf1021fd9104e08951bd81c1946b074, server=a23eb99f71f0,34777,1732468442786 in 11.6640 sec 2024-11-24T17:14:37,130 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/7ba4002d0b6663684be2c126a81b2c74/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,131 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened a61743f7cd3883457b967d55f98541e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61447297, jitterRate=-0.0843639224767685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,131 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:37,131 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for a61743f7cd3883457b967d55f98541e1: Running coprocessor pre-open hook at 1732468476872Writing region info on filesystem at 1732468476872Initializing all the Stores at 1732468476909 (+37 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468476909Cleaning up temporary data from old regions at 1732468477042 (+133 ms)Running coprocessor post-open hooks at 1732468477131 (+89 ms)Region opened successfully at 1732468477131 2024-11-24T17:14:37,132 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 7ba4002d0b6663684be2c126a81b2c74; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63585245, jitterRate=-0.052506014704704285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,133 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:37,133 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 7ba4002d0b6663684be2c126a81b2c74: Running coprocessor pre-open hook at 1732468476929Writing region info on filesystem at 1732468476929Initializing all the Stores at 1732468476935 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468476935Cleaning up temporary data from old regions at 1732468477045 (+110 ms)Running coprocessor post-open hooks at 1732468477133 (+88 ms)Region opened successfully at 1732468477133 2024-11-24T17:14:37,135 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74., pid=38, masterSystemTime=1732468465506 2024-11-24T17:14:37,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=acf1021fd9104e08951bd81c1946b074, ASSIGN in 12.0610 sec 2024-11-24T17:14:37,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:37,137 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1., pid=40, masterSystemTime=1732468465504 2024-11-24T17:14:37,139 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:37,139 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:37,139 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:37,139 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 4707e344fe9a8cd93af441dfe2b9492e, NAME => 'test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-24T17:14:37,140 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,140 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,140 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,140 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,143 INFO [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718/a23eb99f71f0%2C38279%2C1732468442718.1732468446724 with entries=4, filesize=1.13 KB; new WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718/a23eb99f71f0%2C38279%2C1732468442718.1732468476854 2024-11-24T17:14:37,147 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:37,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:37,147 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:37,147 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:37,148 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 67115580ab5444880122d14f983db95c, NAME => 'test,www,1732468449655.67115580ab5444880122d14f983db95c.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-24T17:14:37,148 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,148 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,www,1732468449655.67115580ab5444880122d14f983db95c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,148 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,148 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,151 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:37,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure a61743f7cd3883457b967d55f98541e1, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:37,154 INFO [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.1732468446752 with entries=5, filesize=1.39 KB; new WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.1732468476851 2024-11-24T17:14:37,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ba4002d0b6663684be2c126a81b2c74, server=a23eb99f71f0,34777,1732468442786 because future has completed 2024-11-24T17:14:37,157 INFO [StoreOpener-4707e344fe9a8cd93af441dfe2b9492e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,159 INFO [StoreOpener-4707e344fe9a8cd93af441dfe2b9492e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4707e344fe9a8cd93af441dfe2b9492e columnFamilyName family 2024-11-24T17:14:37,159 DEBUG [StoreOpener-4707e344fe9a8cd93af441dfe2b9492e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,160 INFO [StoreOpener-4707e344fe9a8cd93af441dfe2b9492e-1 {}] regionserver.HStore(327): Store=4707e344fe9a8cd93af441dfe2b9492e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,160 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=16 2024-11-24T17:14:37,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure a61743f7cd3883457b967d55f98541e1, server=a23eb99f71f0,38279,1732468442718 in 11.7570 sec 2024-11-24T17:14:37,162 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,162 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,163 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,163 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,163 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/WALs/a23eb99f71f0,40213,1732468441272/a23eb99f71f0%2C40213%2C1732468441272.1732468444106 with entries=297, filesize=149.71 KB; new WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/WALs/a23eb99f71f0,40213,1732468441272/a23eb99f71f0%2C40213%2C1732468441272.1732468476856 2024-11-24T17:14:37,165 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,176 DEBUG [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43517:43517),(127.0.0.1/127.0.0.1:46283:46283),(127.0.0.1/127.0.0.1:46053:46053)] 2024-11-24T17:14:37,176 DEBUG [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718/a23eb99f71f0%2C38279%2C1732468442718.1732468446724 is not closed yet, will try archiving it next time 2024-11-24T17:14:37,176 INFO [StoreOpener-67115580ab5444880122d14f983db95c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,177 DEBUG [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43517:43517),(127.0.0.1/127.0.0.1:46283:46283),(127.0.0.1/127.0.0.1:46053:46053)] 2024-11-24T17:14:37,177 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0., pid=46, masterSystemTime=1732468465619 2024-11-24T17:14:37,177 DEBUG [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.1732468446752 is not closed yet, will try archiving it next time 2024-11-24T17:14:37,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741834_1010 (size=1165) 2024-11-24T17:14:37,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741835_1011 (size=1433) 2024-11-24T17:14:37,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741834_1010 (size=1165) 2024-11-24T17:14:37,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741834_1010 (size=1165) 2024-11-24T17:14:37,181 INFO [StoreOpener-67115580ab5444880122d14f983db95c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67115580ab5444880122d14f983db95c columnFamilyName family 2024-11-24T17:14:37,181 DEBUG [StoreOpener-67115580ab5444880122d14f983db95c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,182 INFO [StoreOpener-67115580ab5444880122d14f983db95c-1 {}] regionserver.HStore(327): Store=67115580ab5444880122d14f983db95c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,182 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741835_1011 (size=1433) 2024-11-24T17:14:37,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741835_1011 (size=1433) 2024-11-24T17:14:37,184 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,185 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,186 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,186 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,189 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,194 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4707e344fe9a8cd93af441dfe2b9492e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,195 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 4707e344fe9a8cd93af441dfe2b9492e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65514507, jitterRate=-0.023757770657539368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,196 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:37,196 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 4707e344fe9a8cd93af441dfe2b9492e: Running coprocessor pre-open hook at 1732468477140Writing region info on filesystem at 1732468477140Initializing all the Stores at 1732468477143 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477143Cleaning up temporary data from old regions at 1732468477163 (+20 ms)Running coprocessor post-open hooks at 1732468477196 (+33 ms)Region opened successfully at 1732468477196 2024-11-24T17:14:37,198 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e., pid=52, masterSystemTime=1732468465866 2024-11-24T17:14:37,214 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43517:43517),(127.0.0.1/127.0.0.1:46053:46053),(127.0.0.1/127.0.0.1:46283:46283)] 2024-11-24T17:14:37,214 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/WALs/a23eb99f71f0,40213,1732468441272/a23eb99f71f0%2C40213%2C1732468441272.1732468444106 is not closed yet, will try archiving it next time 2024-11-24T17:14:37,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a61743f7cd3883457b967d55f98541e1, ASSIGN in 12.0930 sec 2024-11-24T17:14:37,216 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,216 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=7ba4002d0b6663684be2c126a81b2c74. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,217 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:37,217 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741830_1006 (size=153316) 2024-11-24T17:14:37,217 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741830_1006 (size=153316) 2024-11-24T17:14:37,217 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 12eb6cfa7992f8b2d57701a44d5e5526, NAME => 'test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-24T17:14:37,218 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,218 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,218 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,218 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,219 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=3963c202e600d921e58afb7a1966f4f0, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:37,220 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:37,220 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:37,220 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:37,220 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => d888fda423b365786e1d269df49d1cfd, NAME => 'test,,1732468449655.d888fda423b365786e1d269df49d1cfd.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-24T17:14:37,221 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,221 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,,1732468449655.d888fda423b365786e1d269df49d1cfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,221 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,221 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,224 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596/a23eb99f71f0%2C44781%2C1732468442596.1732468446752 to hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs/a23eb99f71f0%2C44781%2C1732468442596.1732468446752 2024-11-24T17:14:37,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:37,225 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,38279,1732468442718/a23eb99f71f0%2C38279%2C1732468442718.1732468446724 to hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs/a23eb99f71f0%2C38279%2C1732468442718.1732468446724 2024-11-24T17:14:37,226 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:37,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4707e344fe9a8cd93af441dfe2b9492e, server=a23eb99f71f0,34777,1732468442786 because future has completed 2024-11-24T17:14:37,234 INFO [StoreOpener-12eb6cfa7992f8b2d57701a44d5e5526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,237 INFO [StoreOpener-d888fda423b365786e1d269df49d1cfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,318 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=3963c202e600d921e58afb7a1966f4f0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,318 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,319 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=4707e344fe9a8cd93af441dfe2b9492e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,319 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=7ba4002d0b6663684be2c126a81b2c74. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,421 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=4707e344fe9a8cd93af441dfe2b9492e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,421 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=7ba4002d0b6663684be2c126a81b2c74. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,422 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=3963c202e600d921e58afb7a1966f4f0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,422 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,434 INFO [AsyncFSWAL-0-hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData-prefix:a23eb99f71f0,40213,1732468441272 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK], DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK], DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK]] 2024-11-24T17:14:37,436 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/67115580ab5444880122d14f983db95c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,436 INFO [StoreOpener-d888fda423b365786e1d269df49d1cfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d888fda423b365786e1d269df49d1cfd columnFamilyName family 2024-11-24T17:14:37,436 DEBUG [StoreOpener-d888fda423b365786e1d269df49d1cfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,436 INFO [StoreOpener-12eb6cfa7992f8b2d57701a44d5e5526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12eb6cfa7992f8b2d57701a44d5e5526 columnFamilyName family 2024-11-24T17:14:37,437 DEBUG [StoreOpener-12eb6cfa7992f8b2d57701a44d5e5526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,438 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 67115580ab5444880122d14f983db95c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75406492, jitterRate=0.12364429235458374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,438 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 67115580ab5444880122d14f983db95c 2024-11-24T17:14:37,438 INFO [StoreOpener-d888fda423b365786e1d269df49d1cfd-1 {}] regionserver.HStore(327): Store=d888fda423b365786e1d269df49d1cfd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,439 INFO [StoreOpener-12eb6cfa7992f8b2d57701a44d5e5526-1 {}] regionserver.HStore(327): Store=12eb6cfa7992f8b2d57701a44d5e5526/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,439 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 67115580ab5444880122d14f983db95c: Running coprocessor pre-open hook at 1732468477149Writing region info on filesystem at 1732468477149Initializing all the Stores at 1732468477151 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477151Cleaning up temporary data from old regions at 1732468477186 (+35 ms)Running coprocessor post-open hooks at 1732468477438 (+252 ms)Region opened successfully at 1732468477439 (+1 ms) 2024-11-24T17:14:37,439 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,439 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,440 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,441 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,441 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,442 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,442 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,442 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,442 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732468449655.67115580ab5444880122d14f983db95c., pid=43, masterSystemTime=1732468465504 2024-11-24T17:14:37,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=13 2024-11-24T17:14:37,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,44781,1732468442596 in 11.9700 sec 2024-11-24T17:14:37,445 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,445 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,446 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:37,446 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:37,447 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:37,447 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 3bf176f82dd506ad26a9f1d6ebd6f33a, NAME => 'test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-24T17:14:37,447 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,448 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,448 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,448 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,449 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,450 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,454 INFO [StoreOpener-3bf176f82dd506ad26a9f1d6ebd6f33a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN in 12.3760 sec 2024-11-24T17:14:37,458 INFO [StoreOpener-3bf176f82dd506ad26a9f1d6ebd6f33a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bf176f82dd506ad26a9f1d6ebd6f33a columnFamilyName family 2024-11-24T17:14:37,458 DEBUG [StoreOpener-3bf176f82dd506ad26a9f1d6ebd6f33a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,459 INFO [StoreOpener-3bf176f82dd506ad26a9f1d6ebd6f33a-1 {}] regionserver.HStore(327): Store=3bf176f82dd506ad26a9f1d6ebd6f33a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,459 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,460 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/12eb6cfa7992f8b2d57701a44d5e5526/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,461 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,462 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,462 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 12eb6cfa7992f8b2d57701a44d5e5526; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66917569, jitterRate=-0.0028505176305770874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,462 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:37,462 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 12eb6cfa7992f8b2d57701a44d5e5526: Running coprocessor pre-open hook at 1732468477219Writing region info on filesystem at 1732468477219Initializing all the Stores at 1732468477222 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477222Cleaning up temporary data from old regions at 1732468477442 (+220 ms)Running coprocessor post-open hooks at 1732468477462 (+20 ms)Region opened successfully at 1732468477462 2024-11-24T17:14:37,463 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,463 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,465 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526., pid=47, masterSystemTime=1732468465619 2024-11-24T17:14:37,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=67115580ab5444880122d14f983db95c, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:37,468 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,468 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/d888fda423b365786e1d269df49d1cfd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,469 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened d888fda423b365786e1d269df49d1cfd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70272790, jitterRate=0.04714617133140564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,469 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:37,469 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for d888fda423b365786e1d269df49d1cfd: Running coprocessor pre-open hook at 1732468477221Writing region info on filesystem at 1732468477221Initializing all the Stores at 1732468477225 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477225Cleaning up temporary data from old regions at 1732468477445 (+220 ms)Running coprocessor post-open hooks at 1732468477469 (+24 ms)Region opened successfully at 1732468477469 2024-11-24T17:14:37,471 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:37,471 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732468449655.d888fda423b365786e1d269df49d1cfd., pid=54, masterSystemTime=1732468477169 2024-11-24T17:14:37,471 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:37,471 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. 2024-11-24T17:14:37,471 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b5321b4dd571156ebac77ca1fa8b150, NAME => 'test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-24T17:14:37,472 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,472 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,472 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,472 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67115580ab5444880122d14f983db95c, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:37,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:37,476 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:37,476 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:37,476 INFO [StoreOpener-4b5321b4dd571156ebac77ca1fa8b150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,477 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3bf176f82dd506ad26a9f1d6ebd6f33a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,478 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 3bf176f82dd506ad26a9f1d6ebd6f33a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64125628, jitterRate=-0.04445368051528931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12eb6cfa7992f8b2d57701a44d5e5526, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:37,479 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:37,479 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 3bf176f82dd506ad26a9f1d6ebd6f33a: Running coprocessor pre-open hook at 1732468477448Writing region info on filesystem at 1732468477448Initializing all the Stores at 1732468477450 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477452 (+2 ms)Cleaning up temporary data from old regions at 1732468477463 (+11 ms)Running coprocessor post-open hooks at 1732468477479 (+16 ms)Region opened successfully at 1732468477479 2024-11-24T17:14:37,480 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:37,480 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a., pid=34, masterSystemTime=1732468465504 2024-11-24T17:14:37,481 INFO [StoreOpener-4b5321b4dd571156ebac77ca1fa8b150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5321b4dd571156ebac77ca1fa8b150 columnFamilyName family 2024-11-24T17:14:37,481 DEBUG [StoreOpener-4b5321b4dd571156ebac77ca1fa8b150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,483 INFO [StoreOpener-4b5321b4dd571156ebac77ca1fa8b150-1 {}] regionserver.HStore(327): Store=4b5321b4dd571156ebac77ca1fa8b150/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,483 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:37,483 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:37,483 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:37,484 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => b1ebd571dd5d321046e5067f5b19ad97, NAME => 'test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-24T17:14:37,484 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,484 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,484 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,484 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,486 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,488 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,488 INFO [StoreOpener-b1ebd571dd5d321046e5067f5b19ad97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d888fda423b365786e1d269df49d1cfd, server=a23eb99f71f0,34777,1732468442786 because future has completed 2024-11-24T17:14:37,489 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=17 2024-11-24T17:14:37,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 12eb6cfa7992f8b2d57701a44d5e5526, server=a23eb99f71f0,44781,1732468442596 in 12.0110 sec 2024-11-24T17:14:37,489 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,489 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,490 INFO [StoreOpener-b1ebd571dd5d321046e5067f5b19ad97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1ebd571dd5d321046e5067f5b19ad97 columnFamilyName family 2024-11-24T17:14:37,490 DEBUG [StoreOpener-b1ebd571dd5d321046e5067f5b19ad97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,491 INFO [StoreOpener-b1ebd571dd5d321046e5067f5b19ad97-1 {}] regionserver.HStore(327): Store=b1ebd571dd5d321046e5067f5b19ad97/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,491 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,492 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,492 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN in 12.4210 sec 2024-11-24T17:14:37,493 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,493 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,493 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,496 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,496 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4b5321b4dd571156ebac77ca1fa8b150/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,497 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 4b5321b4dd571156ebac77ca1fa8b150; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61904080, jitterRate=-0.07755732536315918}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,497 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:37,497 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 4b5321b4dd571156ebac77ca1fa8b150: Running coprocessor pre-open hook at 1732468477472Writing region info on filesystem at 1732468477472Initializing all the Stores at 1732468477476 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477476Cleaning up temporary data from old regions at 1732468477489 (+13 ms)Running coprocessor post-open hooks at 1732468477497 (+8 ms)Region opened successfully at 1732468477497 2024-11-24T17:14:37,499 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150., pid=50, masterSystemTime=1732468465839 2024-11-24T17:14:37,500 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/b1ebd571dd5d321046e5067f5b19ad97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,501 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened b1ebd571dd5d321046e5067f5b19ad97; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70157163, jitterRate=0.04542319476604462}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,501 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:37,501 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for b1ebd571dd5d321046e5067f5b19ad97: Running coprocessor pre-open hook at 1732468477485Writing region info on filesystem at 1732468477485Initializing all the Stores at 1732468477486 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477486Cleaning up temporary data from old regions at 1732468477493 (+7 ms)Running coprocessor post-open hooks at 1732468477501 (+8 ms)Region opened successfully at 1732468477501 2024-11-24T17:14:37,509 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97., pid=41, masterSystemTime=1732468465504 2024-11-24T17:14:37,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=5 2024-11-24T17:14:37,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d888fda423b365786e1d269df49d1cfd, server=a23eb99f71f0,34777,1732468442786 in 483 msec 2024-11-24T17:14:37,509 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. 2024-11-24T17:14:37,510 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. 2024-11-24T17:14:37,510 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:37,510 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a1b996a2c1476427750ae2fbe015bc5, NAME => 'test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-24T17:14:37,510 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,510 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,511 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,511 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:37,512 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:37,512 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:37,512 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:37,512 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 448f6154a795fbeb01b36798bcbacb8f, NAME => 'test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-24T17:14:37,513 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,513 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,513 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,513 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,514 INFO [StoreOpener-4a1b996a2c1476427750ae2fbe015bc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:37,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:37,519 INFO [StoreOpener-4a1b996a2c1476427750ae2fbe015bc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1b996a2c1476427750ae2fbe015bc5 columnFamilyName family 2024-11-24T17:14:37,519 DEBUG [StoreOpener-4a1b996a2c1476427750ae2fbe015bc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,520 INFO [StoreOpener-4a1b996a2c1476427750ae2fbe015bc5-1 {}] regionserver.HStore(327): Store=4a1b996a2c1476427750ae2fbe015bc5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,520 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,521 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,522 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,523 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,523 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,524 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=4b5321b4dd571156ebac77ca1fa8b150. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,524 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1ebd571dd5d321046e5067f5b19ad97, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:37,527 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=4707e344fe9a8cd93af441dfe2b9492e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,527 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=7ba4002d0b6663684be2c126a81b2c74. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,528 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,38279,1732468442718, table=test, region=b1ebd571dd5d321046e5067f5b19ad97. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,528 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,38279,1732468442718, table=test, region=67115580ab5444880122d14f983db95c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,530 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,534 INFO [StoreOpener-448f6154a795fbeb01b36798bcbacb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=7 2024-11-24T17:14:37,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,44781,1732468442596 in 11.8440 sec 2024-11-24T17:14:37,537 INFO [StoreOpener-448f6154a795fbeb01b36798bcbacb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 448f6154a795fbeb01b36798bcbacb8f columnFamilyName family 2024-11-24T17:14:37,537 DEBUG [StoreOpener-448f6154a795fbeb01b36798bcbacb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,538 INFO [StoreOpener-448f6154a795fbeb01b36798bcbacb8f-1 {}] regionserver.HStore(327): Store=448f6154a795fbeb01b36798bcbacb8f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,538 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN in 12.4690 sec 2024-11-24T17:14:37,540 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,541 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,542 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,542 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,544 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4a1b996a2c1476427750ae2fbe015bc5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,546 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 4a1b996a2c1476427750ae2fbe015bc5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67885063, jitterRate=0.011566266417503357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,546 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:37,546 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,546 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 4a1b996a2c1476427750ae2fbe015bc5: Running coprocessor pre-open hook at 1732468477511Writing region info on filesystem at 1732468477511Initializing all the Stores at 1732468477513 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477513Cleaning up temporary data from old regions at 1732468477523 (+10 ms)Running coprocessor post-open hooks at 1732468477546 (+23 ms)Region opened successfully at 1732468477546 2024-11-24T17:14:37,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=18 2024-11-24T17:14:37,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure b1ebd571dd5d321046e5067f5b19ad97, server=a23eb99f71f0,38279,1732468442718 in 12.1350 sec 2024-11-24T17:14:37,548 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5., pid=55, masterSystemTime=1732468477293 2024-11-24T17:14:37,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b1ebd571dd5d321046e5067f5b19ad97, ASSIGN in 12.4800 sec 2024-11-24T17:14:37,552 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:37,552 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:37,553 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:37,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:37,557 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/448f6154a795fbeb01b36798bcbacb8f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,558 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 448f6154a795fbeb01b36798bcbacb8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73404462, jitterRate=0.09381172060966492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,558 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:37,558 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 448f6154a795fbeb01b36798bcbacb8f: Running coprocessor pre-open hook at 1732468477513Writing region info on filesystem at 1732468477513Initializing all the Stores at 1732468477515 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477515Cleaning up temporary data from old regions at 1732468477542 (+27 ms)Running coprocessor post-open hooks at 1732468477558 (+16 ms)Region opened successfully at 1732468477558 2024-11-24T17:14:37,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d888fda423b365786e1d269df49d1cfd, ASSIGN in 12.4870 sec 2024-11-24T17:14:37,560 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f., pid=53, masterSystemTime=1732468465850 2024-11-24T17:14:37,562 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:37,562 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:37,562 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:37,562 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 94439407a643ae820e9015188cf23ed6, NAME => 'test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-24T17:14:37,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=8 2024-11-24T17:14:37,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,44781,1732468442596 in 424 msec 2024-11-24T17:14:37,566 INFO [StoreOpener-94439407a643ae820e9015188cf23ed6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN in 12.4960 sec 2024-11-24T17:14:37,568 INFO [StoreOpener-94439407a643ae820e9015188cf23ed6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94439407a643ae820e9015188cf23ed6 columnFamilyName family 2024-11-24T17:14:37,568 DEBUG [StoreOpener-94439407a643ae820e9015188cf23ed6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,569 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:37,569 INFO [StoreOpener-94439407a643ae820e9015188cf23ed6-1 {}] regionserver.HStore(327): Store=94439407a643ae820e9015188cf23ed6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,570 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,572 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,572 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,574 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 448f6154a795fbeb01b36798bcbacb8f, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:37,574 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,576 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:37,576 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bf176f82dd506ad26a9f1d6ebd6f33a, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:37,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=10 2024-11-24T17:14:37,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 448f6154a795fbeb01b36798bcbacb8f, server=a23eb99f71f0,38279,1732468442718 in 11.8560 sec 2024-11-24T17:14:37,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=448f6154a795fbeb01b36798bcbacb8f, ASSIGN in 12.5180 sec 2024-11-24T17:14:37,592 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/94439407a643ae820e9015188cf23ed6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,593 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 94439407a643ae820e9015188cf23ed6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65727254, jitterRate=-0.02058759331703186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,593 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:37,594 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 94439407a643ae820e9015188cf23ed6: Running coprocessor pre-open hook at 1732468477563Writing region info on filesystem at 1732468477563Initializing all the Stores at 1732468477565 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477565Cleaning up temporary data from old regions at 1732468477574 (+9 ms)Running coprocessor post-open hooks at 1732468477593 (+19 ms)Region opened successfully at 1732468477593 2024-11-24T17:14:37,595 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6., pid=51, masterSystemTime=1732468465850 2024-11-24T17:14:37,597 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:37,597 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:37,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=20 2024-11-24T17:14:37,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 3bf176f82dd506ad26a9f1d6ebd6f33a, server=a23eb99f71f0,38279,1732468442718 in 12.2540 sec 2024-11-24T17:14:37,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3bf176f82dd506ad26a9f1d6ebd6f33a, ASSIGN in 12.5320 sec 2024-11-24T17:14:37,609 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=94439407a643ae820e9015188cf23ed6, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:37,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94439407a643ae820e9015188cf23ed6, server=a23eb99f71f0,38279,1732468442718 because future has completed 2024-11-24T17:14:37,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=27 2024-11-24T17:14:37,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 67115580ab5444880122d14f983db95c, server=a23eb99f71f0,38279,1732468442718 in 12.1830 sec 2024-11-24T17:14:37,627 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,44781,1732468442596, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=67115580ab5444880122d14f983db95c, ASSIGN in 12.5520 sec 2024-11-24T17:14:37,631 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,38279,1732468442718, table=test, region=94439407a643ae820e9015188cf23ed6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,634 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=4707e344fe9a8cd93af441dfe2b9492e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,634 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=test, region=7ba4002d0b6663684be2c126a81b2c74. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:37,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=6 2024-11-24T17:14:37,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 94439407a643ae820e9015188cf23ed6, server=a23eb99f71f0,38279,1732468442718 in 11.9390 sec 2024-11-24T17:14:37,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=94439407a643ae820e9015188cf23ed6, ASSIGN in 12.5690 sec 2024-11-24T17:14:37,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=11 2024-11-24T17:14:37,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 4707e344fe9a8cd93af441dfe2b9492e, server=a23eb99f71f0,34777,1732468442786 in 11.9330 sec 2024-11-24T17:14:37,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4707e344fe9a8cd93af441dfe2b9492e, ASSIGN in 12.5770 sec 2024-11-24T17:14:37,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=22 2024-11-24T17:14:37,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ba4002d0b6663684be2c126a81b2c74, server=a23eb99f71f0,34777,1732468442786 in 12.2700 sec 2024-11-24T17:14:37,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7ba4002d0b6663684be2c126a81b2c74, ASSIGN in 12.5860 sec 2024-11-24T17:14:37,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:37,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=24 2024-11-24T17:14:37,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,44781,1732468442596 in 12.3510 sec 2024-11-24T17:14:37,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN in 12.6000 sec 2024-11-24T17:14:37,815 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:37,815 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => e88c3c1672dcd6a9395b368267add589, NAME => 'test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-24T17:14:37,815 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,815 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:37,816 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,816 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,817 INFO [StoreOpener-e88c3c1672dcd6a9395b368267add589-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,819 INFO [StoreOpener-e88c3c1672dcd6a9395b368267add589-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e88c3c1672dcd6a9395b368267add589 columnFamilyName family 2024-11-24T17:14:37,819 DEBUG [StoreOpener-e88c3c1672dcd6a9395b368267add589-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:37,820 INFO [StoreOpener-e88c3c1672dcd6a9395b368267add589-1 {}] regionserver.HStore(327): Store=e88c3c1672dcd6a9395b368267add589/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:37,820 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,821 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,822 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,822 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,822 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,824 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,827 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/e88c3c1672dcd6a9395b368267add589/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T17:14:37,827 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened e88c3c1672dcd6a9395b368267add589; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62135391, jitterRate=-0.07411052286624908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:37,828 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:37,828 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for e88c3c1672dcd6a9395b368267add589: Running coprocessor pre-open hook at 1732468477816Writing region info on filesystem at 1732468477816Initializing all the Stores at 1732468477817 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468477817Cleaning up temporary data from old regions at 1732468477822 (+5 ms)Running coprocessor post-open hooks at 1732468477828 (+6 ms)Region opened successfully at 1732468477828 2024-11-24T17:14:37,829 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589., pid=56, masterSystemTime=1732468477810 2024-11-24T17:14:37,831 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:37,832 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:37,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e88c3c1672dcd6a9395b368267add589, regionState=OPEN, openSeqNum=2, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:37,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:37,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=9 2024-11-24T17:14:37,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,44781,1732468442596 in 182 msec 2024-11-24T17:14:37,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=4 2024-11-24T17:14:37,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN in 12.7740 sec 2024-11-24T17:14:37,847 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T17:14:37,847 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732468477847"}]},"ts":"1732468477847"} 2024-11-24T17:14:37,850 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-24T17:14:37,851 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T17:14:37,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 28.1790 sec 2024-11-24T17:14:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T17:14:38,012 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-24T17:14:38,013 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-24T17:14:38,014 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:14:38,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-24T17:14:38,034 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:14:38,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-24T17:14:38,036 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:14:38,043 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-24T17:14:38,068 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-24T17:14:38,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T17:14:38,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T17:14:38,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T17:14:38,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T17:14:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d888fda423b365786e1d269df49d1cfd changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4707e344fe9a8cd93af441dfe2b9492e changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3132e986d91cd07c924412b1164b0ada changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region acf1021fd9104e08951bd81c1946b074 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 68cc51974c64ffdbd8bb6b53a87450da changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7ba4002d0b6663684be2c126a81b2c74 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 71e89563154e9bf392a9193da474463d changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 61f230c444c764032b318ed0a81000ab changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 94439407a643ae820e9015188cf23ed6 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 448f6154a795fbeb01b36798bcbacb8f changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 89f671091abaef7d04c744da188c0bf9 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a61743f7cd3883457b967d55f98541e1 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b1ebd571dd5d321046e5067f5b19ad97 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3bf176f82dd506ad26a9f1d6ebd6f33a changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4ed93b4907aeb35b406e6de08e103616 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 67115580ab5444880122d14f983db95c changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4f35eb03f4e248bda613971e436befd3 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4b5321b4dd571156ebac77ca1fa8b150 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4a1b996a2c1476427750ae2fbe015bc5 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e88c3c1672dcd6a9395b368267add589 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3963c202e600d921e58afb7a1966f4f0 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 12eb6cfa7992f8b2d57701a44d5e5526 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a6d2969d9f925c5b8cb0ff244090dcd4 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6f16dc15c0e0aff7efc028181a4fdbe3 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 93d787842f2c5c53adaf00e6e90673e7 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 58c2a35cef97d825f565d67ca7006c36 changed from -1.0 to 0.0, refreshing cache 2024-11-24T17:14:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-24T17:14:38,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-24T17:14:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-24T17:14:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(204): Hosts are {a23eb99f71f0=0} racks are {/default-rack=0} 2024-11-24T17:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(303): server 0 has 8 regions 2024-11-24T17:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-24T17:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(303): server 2 has 10 regions 2024-11-24T17:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T17:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T17:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T17:14:38,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T17:14:38,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T17:14:38,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T17:14:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-24T17:14:38,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-24T17:14:38,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-24T17:14:38,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 519 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-24T17:14:38,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-24T17:14:38,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.HMaster(2172): balance hri=a6d2969d9f925c5b8cb0ff244090dcd4, source=a23eb99f71f0,44781,1732468442596, destination=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:38,697 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:38,699 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=CLOSING, regionLocation=a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:38,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE because future has completed 2024-11-24T17:14:38,703 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T17:14:38,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,44781,1732468442596}] 2024-11-24T17:14:38,860 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:38,861 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T17:14:38,861 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing a6d2969d9f925c5b8cb0ff244090dcd4, disabling compactions & flushes 2024-11-24T17:14:38,861 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:38,861 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:38,861 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. after waiting 0 ms 2024-11-24T17:14:38,861 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:38,867 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a6d2969d9f925c5b8cb0ff244090dcd4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:38,869 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:38,869 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for a6d2969d9f925c5b8cb0ff244090dcd4: Waiting for close lock at 1732468478861Running coprocessor pre-close hooks at 1732468478861Disabling compacts and flushes for region at 1732468478861Disabling writes for close at 1732468478861Writing region close event to WAL at 1732468478862 (+1 ms)Running coprocessor post-close hooks at 1732468478867 (+5 ms)Closed at 1732468478869 (+2 ms) 2024-11-24T17:14:38,870 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding a6d2969d9f925c5b8cb0ff244090dcd4 move to a23eb99f71f0,34777,1732468442786 record at close sequenceid=2 2024-11-24T17:14:38,872 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:38,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=CLOSED 2024-11-24T17:14:38,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,44781,1732468442596 because future has completed 2024-11-24T17:14:38,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-24T17:14:38,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,44781,1732468442596 in 174 msec 2024-11-24T17:14:38,880 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE; state=CLOSED, location=a23eb99f71f0,34777,1732468442786; forceNewPlan=false, retain=false 2024-11-24T17:14:39,031 INFO [a23eb99f71f0:40213 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T17:14:39,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:39,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE because future has completed 2024-11-24T17:14:39,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:39,192 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:39,192 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => a6d2969d9f925c5b8cb0ff244090dcd4, NAME => 'test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-24T17:14:39,193 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,193 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:39,193 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,193 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,195 INFO [StoreOpener-a6d2969d9f925c5b8cb0ff244090dcd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,196 INFO [StoreOpener-a6d2969d9f925c5b8cb0ff244090dcd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6d2969d9f925c5b8cb0ff244090dcd4 columnFamilyName family 2024-11-24T17:14:39,196 DEBUG [StoreOpener-a6d2969d9f925c5b8cb0ff244090dcd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:39,197 INFO [StoreOpener-a6d2969d9f925c5b8cb0ff244090dcd4-1 {}] regionserver.HStore(327): Store=a6d2969d9f925c5b8cb0ff244090dcd4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:39,197 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,198 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,200 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,200 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,200 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,202 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,203 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened a6d2969d9f925c5b8cb0ff244090dcd4; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61897216, jitterRate=-0.07765960693359375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:39,203 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:39,204 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for a6d2969d9f925c5b8cb0ff244090dcd4: Running coprocessor pre-open hook at 1732468479193Writing region info on filesystem at 1732468479193Initializing all the Stores at 1732468479194 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468479195 (+1 ms)Cleaning up temporary data from old regions at 1732468479200 (+5 ms)Running coprocessor post-open hooks at 1732468479203 (+3 ms)Region opened successfully at 1732468479204 (+1 ms) 2024-11-24T17:14:39,205 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4., pid=59, masterSystemTime=1732468479187 2024-11-24T17:14:39,207 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:39,207 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:39,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, openSeqNum=5, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:39,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,34777,1732468442786 because future has completed 2024-11-24T17:14:39,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-24T17:14:39,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure a6d2969d9f925c5b8cb0ff244090dcd4, server=a23eb99f71f0,34777,1732468442786 in 184 msec 2024-11-24T17:14:39,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE in 527 msec 2024-11-24T17:14:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-24T17:14:39,299 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:14:39,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-24T17:14:39,326 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=389, OpenFileDescriptor=629, MaxFileDescriptor=1048576, SystemLoadAverage=1115, ProcessCount=11, AvailableMemoryMB=5164 2024-11-24T17:14:39,329 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-24T17:14:39,330 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-24T17:14:39,331 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:39,331 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server a23eb99f71f0,44781,1732468442596: Simulated kill ***** 2024-11-24T17:14:39,331 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T17:14:39,334 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T17:14:39,335 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T17:14:39,338 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T17:14:39,339 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T17:14:39,346 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 475145440 }, "NonHeapMemoryUsage": { "committed": 159252480, "init": 7667712, "max": -1, "used": 155956392 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "a23eb99f71f0", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 22164, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 51, "ProcessCallTime_min": 0, "ProcessCallTime_max": 0, "ProcessCallTime_mean": 0, "ProcessCallTime_25th_percentile": 0, "ProcessCallTime_median": 0, "ProcessCallTime_75th_percentile": 0, "ProcessCallTime_90th_percentile": 0, "ProcessCallTime_95th_percentile": 0, "ProcessCallTime_98th_percentile": 0, "ProcessCallTime_99th_percentile": 0, "ProcessCallTime_99.9th_percentile": 0, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 51, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 51, "TotalCallTime_min": 0, "TotalCallTime_max": 0, "TotalCallTime_mean": 0, "TotalCallTime_25th_percentile": 0, "TotalCallTime_median": 0, "TotalCallTime_75th_percentile": 0, "TotalCallTime_90th_percentile": 0, "TotalCallTime_95th_percentile": 0, "TotalCallTime_98th_percentile": 0, "TotalCallTime_99th_percentile": 0, "TotalCallTime_99.9th_percentile": 0, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 51, "ResponseSize_min": 0, "ResponseSize_max": 0, "ResponseSize_mean": 0, "ResponseSize_25th_percentile": 0, "ResponseSize_median": 0, "ResponseSize_75th_percentile": 0, "ResponseSize_90th_percentile": 0, "ResponseSize_95th_percentile": 0, "ResponseSize_98th_percentile": 0, "ResponseSize_99th_percentile": 0, "ResponseSize_99.9th_percentile": 0, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 51, "RequestSize_min": 0, "RequestSize_max": 0, "RequestSize_mean": 0, "RequestSize_25th_percentile": 0, "RequestSize_median": 0, "RequestSize_75th_percentile": 0, "RequestSize_90th_percentile": 0, "RequestSize_95th_percentile": 0, "RequestSize_98th_percentile": 0, "RequestSize_99th_percentile": 0, "RequestSize_99.9th_percentile": 0, "sentBytes": 924 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "a23eb99f71f0", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52152", "tag.serverName": "a23eb99f71f0,44781,1732468442596", "tag.clusterId": "866d63e0-137b-480d-bc99-aace7e6a98b3", "tag.Context": "regionserver", "tag.Hostname": "a23eb99f71f0", "regionCount": 4, "storeCount": 7, "hlogFileCount": 8, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 19251, "memStoreHeapSize": 34224, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732468442596, "averageRegionSize": 4812, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 12.200000000000001, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 43, "totalRowActionRequestCount": 70, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 63, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 37, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 37, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T17:14:39,350 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40213 {}] master.MasterRpcServices(700): a23eb99f71f0,44781,1732468442596 reported a fatal error: ***** ABORTING region server a23eb99f71f0,44781,1732468442596: Simulated kill ***** 2024-11-24T17:14:39,353 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a23eb99f71f0,44781,1732468442596' ***** 2024-11-24T17:14:39,353 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-24T17:14:39,353 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T17:14:39,354 INFO [RS:0;a23eb99f71f0:44781 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T17:14:39,354 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T17:14:39,354 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:14:39,354 INFO [RS:0;a23eb99f71f0:44781 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T17:14:39,354 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T17:14:39,354 INFO [RS:0;a23eb99f71f0:44781 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a23eb99f71f0:44781. 2024-11-24T17:14:39,354 DEBUG [RS:0;a23eb99f71f0:44781 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:14:39,362 DEBUG [RS:0;a23eb99f71f0:44781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:39,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741836_1012 (size=33749) 2024-11-24T17:14:39,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741836_1012 (size=33749) 2024-11-24T17:14:39,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741836_1012 (size=33749) 2024-11-24T17:14:39,394 INFO [regionserver/a23eb99f71f0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T17:14:39,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741864_1040 (size=1434) 2024-11-24T17:14:39,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741864_1040 (size=1434) 2024-11-24T17:14:39,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741864_1040 (size=1434) 2024-11-24T17:14:39,401 DEBUG [RS:0;a23eb99f71f0:44781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:39,401 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T17:14:39,401 INFO [RS:0;a23eb99f71f0:44781 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44781 2024-11-24T17:14:39,494 INFO [regionserver/a23eb99f71f0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T17:14:39,494 INFO [regionserver/a23eb99f71f0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T17:14:39,607 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T17:14:39,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a23eb99f71f0,44781,1732468442596 2024-11-24T17:14:39,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T17:14:39,655 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a23eb99f71f0,44781,1732468442596] 2024-11-24T17:14:39,708 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a23eb99f71f0,44781,1732468442596 already deleted, retry=false 2024-11-24T17:14:39,709 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a23eb99f71f0,44781,1732468442596 on a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:39,713 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:39,719 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for a23eb99f71f0,44781,1732468442596 (carryingMeta=true) a23eb99f71f0,44781,1732468442596/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1a29bb95[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T17:14:39,719 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:39,720 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true, isMeta: true 2024-11-24T17:14:39,722 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting 2024-11-24T17:14:39,723 INFO [PEWorker-1 {}] master.SplitWALManager(105): a23eb99f71f0,44781,1732468442596 WAL count=1, meta=true 2024-11-24T17:14:39,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta}] 2024-11-24T17:14:39,731 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:39,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:39,756 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T17:14:39,756 INFO [RS:0;a23eb99f71f0:44781 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T17:14:39,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:14:39,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x1016dd6789d0001, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:14:39,756 INFO [RS:0;a23eb99f71f0:44781 {}] regionserver.HRegionServer(1031): Exiting; stopping=a23eb99f71f0,44781,1732468442596; zookeeper connection closed. 2024-11-24T17:14:39,756 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1b88c2c0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1b88c2c0 2024-11-24T17:14:39,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38279 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-24T17:14:39,915 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, size=33.0 K (33749bytes) 2024-11-24T17:14:39,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T17:14:39,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T17:14:39,916 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta 2024-11-24T17:14:39,920 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta after 3ms 2024-11-24T17:14:39,927 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta took 13ms 2024-11-24T17:14:39,937 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-24T17:14:39,944 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta.temp 2024-11-24T17:14:39,948 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta.temp 2024-11-24T17:14:39,949 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta so closing down 2024-11-24T17:14:39,949 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T17:14:39,953 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T17:14:39,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741867_1043 (size=33508) 2024-11-24T17:14:39,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741867_1043 (size=33508) 2024-11-24T17:14:39,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741867_1043 (size=33508) 2024-11-24T17:14:39,974 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-24T17:14:39,976 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta.temp to hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-24T17:14:39,980 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 48 ms; skipped=1; WAL=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-24T17:14:39,980 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, journal: Splitting hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, size=33.0 K (33749bytes) at 1732468479915Creating recovered edits writer path=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta.temp at 1732468479948 (+33 ms)Finishing writing output for hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta so closing down at 1732468479949 (+1 ms)3 split writer threads finished at 1732468479953 (+4 ms)Closed recovered edits writer path=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1732468479974 (+21 ms)Rename recovered edits hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta.temp to hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732468479976 (+2 ms)Processed 62 edits across 1 Regions in 48 ms; skipped=1; WAL=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732468479980 (+4 ms) 2024-11-24T17:14:39,981 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta 2024-11-24T17:14:39,982 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-24T17:14:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-24T17:14:39,989 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta to hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs 2024-11-24T17:14:39,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-24T17:14:39,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 in 256 msec 2024-11-24T17:14:39,994 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:39,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T17:14:39,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 in 269 msec 2024-11-24T17:14:39,998 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting dir is empty, no logs to split. 2024-11-24T17:14:39,999 INFO [PEWorker-1 {}] master.SplitWALManager(105): a23eb99f71f0,44781,1732468442596 WAL count=0, meta=true 2024-11-24T17:14:39,999 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if a23eb99f71f0,44781,1732468442596 WAL splitting is done? wals=0, meta=true 2024-11-24T17:14:40,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T17:14:40,038 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:40,040 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:40,191 DEBUG [a23eb99f71f0:40213 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T17:14:40,191 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(204): Hosts are {a23eb99f71f0=0} racks are {/default-rack=0} 2024-11-24T17:14:40,191 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T17:14:40,191 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T17:14:40,191 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T17:14:40,191 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T17:14:40,191 INFO [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T17:14:40,191 INFO [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T17:14:40,191 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T17:14:40,192 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:40,193 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPENING 2024-11-24T17:14:40,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:40,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:40,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:40,232 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:40,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:40,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:40,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:40,233 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:40,390 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T17:14:40,390 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T17:14:40,390 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T17:14:40,393 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a23eb99f71f0%2C34777%2C1732468442786.meta, suffix=.meta, logDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,34777,1732468442786, archiveDir=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs, maxLogs=32 2024-11-24T17:14:40,408 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,34777,1732468442786/a23eb99f71f0%2C34777%2C1732468442786.meta.1732468480395.meta, exclude list is [], retry=0 2024-11-24T17:14:40,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45925,DS-4d8d1dad-adcf-4a2d-9d12-2ad78c861b9f,DISK] 2024-11-24T17:14:40,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37313,DS-ef9ff03e-aec2-44f8-8062-13057a91ac2b,DISK] 2024-11-24T17:14:40,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37919,DS-c390e229-ba70-42d4-9fa4-8862ae7d9f09,DISK] 2024-11-24T17:14:40,414 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,34777,1732468442786/a23eb99f71f0%2C34777%2C1732468442786.meta.1732468480395.meta 2024-11-24T17:14:40,414 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46053:46053),(127.0.0.1/127.0.0.1:43517:43517),(127.0.0.1/127.0.0.1:46283:46283)] 2024-11-24T17:14:40,415 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T17:14:40,415 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T17:14:40,415 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T17:14:40,415 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T17:14:40,415 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T17:14:40,415 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:40,415 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T17:14:40,415 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T17:14:40,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T17:14:40,418 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T17:14:40,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:40,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:40,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T17:14:40,420 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T17:14:40,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:40,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:40,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T17:14:40,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T17:14:40,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:40,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:40,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T17:14:40,423 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T17:14:40,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:40,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T17:14:40,424 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T17:14:40,425 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740 2024-11-24T17:14:40,427 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740 2024-11-24T17:14:40,428 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-24T17:14:40,443 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-24T17:14:40,447 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-24T17:14:40,520 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/info/1df3d48cacc64c6a892a08f23dd6c39c is 138, key is test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6./info:regioninfo/1732468477608/Put/seqid=0 2024-11-24T17:14:40,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741869_1045 (size=36371) 2024-11-24T17:14:40,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741869_1045 (size=36371) 2024-11-24T17:14:40,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741869_1045 (size=36371) 2024-11-24T17:14:40,535 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/info/1df3d48cacc64c6a892a08f23dd6c39c 2024-11-24T17:14:40,621 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/ns/6c57a253c6a0461dbb58c14309e26877 is 43, key is default/ns:d/1732468449032/Put/seqid=0 2024-11-24T17:14:40,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741870_1046 (size=5153) 2024-11-24T17:14:40,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741870_1046 (size=5153) 2024-11-24T17:14:40,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741870_1046 (size=5153) 2024-11-24T17:14:40,645 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/ns/6c57a253c6a0461dbb58c14309e26877 2024-11-24T17:14:40,679 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/table/60177c88ad5442378ca74a373aa74373 is 40, key is test/table:state/1732468477847/Put/seqid=0 2024-11-24T17:14:40,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741871_1047 (size=5165) 2024-11-24T17:14:40,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741871_1047 (size=5165) 2024-11-24T17:14:40,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741871_1047 (size=5165) 2024-11-24T17:14:40,695 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/table/60177c88ad5442378ca74a373aa74373 2024-11-24T17:14:40,706 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/info/1df3d48cacc64c6a892a08f23dd6c39c as hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/info/1df3d48cacc64c6a892a08f23dd6c39c 2024-11-24T17:14:40,714 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/info/1df3d48cacc64c6a892a08f23dd6c39c, entries=273, sequenceid=64, filesize=35.5 K 2024-11-24T17:14:40,717 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/ns/6c57a253c6a0461dbb58c14309e26877 as hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/ns/6c57a253c6a0461dbb58c14309e26877 2024-11-24T17:14:40,728 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/ns/6c57a253c6a0461dbb58c14309e26877, entries=2, sequenceid=64, filesize=5.0 K 2024-11-24T17:14:40,730 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/table/60177c88ad5442378ca74a373aa74373 as hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/table/60177c88ad5442378ca74a373aa74373 2024-11-24T17:14:40,741 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/table/60177c88ad5442378ca74a373aa74373, entries=2, sequenceid=64, filesize=5.0 K 2024-11-24T17:14:40,746 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 298ms, sequenceid=64, compaction requested=false; wal=null 2024-11-24T17:14:40,748 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-24T17:14:40,750 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T17:14:40,750 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T17:14:40,751 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T17:14:40,754 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T17:14:40,757 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-24T17:14:40,758 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68121338, jitterRate=0.015087038278579712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T17:14:40,758 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T17:14:40,759 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732468480416Writing region info on filesystem at 1732468480416Initializing all the Stores at 1732468480417 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468480417Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468480417Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468480417Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732468480417Obtaining lock to block concurrent updates at 1732468480447 (+30 ms)Preparing flush snapshotting stores in 1588230740 at 1732468480447Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732468480450 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732468480450Flushing 1588230740/info: creating writer at 1732468480452 (+2 ms)Flushing 1588230740/info: appending metadata at 1732468480509 (+57 ms)Flushing 1588230740/info: closing flushed file at 1732468480512 (+3 ms)Flushing 1588230740/ns: creating writer at 1732468480596 (+84 ms)Flushing 1588230740/ns: appending metadata at 1732468480620 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1732468480621 (+1 ms)Flushing 1588230740/table: creating writer at 1732468480659 (+38 ms)Flushing 1588230740/table: appending metadata at 1732468480678 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732468480678Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd7d327: reopening flushed file at 1732468480704 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@409a5e6d: reopening flushed file at 1732468480716 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@620a1eaf: reopening flushed file at 1732468480728 (+12 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 298ms, sequenceid=64, compaction requested=false; wal=null at 1732468480746 (+18 ms)Cleaning up temporary data from old regions at 1732468480750 (+4 ms)Running coprocessor post-open hooks at 1732468480758 (+8 ms)Region opened successfully at 1732468480759 (+1 ms) 2024-11-24T17:14:40,760 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732468480385 2024-11-24T17:14:40,763 DEBUG [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T17:14:40,763 INFO [RS_OPEN_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T17:14:40,767 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:40,769 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:40,804 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T17:14:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:40,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:40,840 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:40,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:40,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:40,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:40,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-24T17:14:40,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 in 608 msec 2024-11-24T17:14:40,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-24T17:14:40,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 808 msec 2024-11-24T17:14:40,852 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): a23eb99f71f0,44781,1732468442596 had 9 regions 2024-11-24T17:14:40,854 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true, isMeta: false 2024-11-24T17:14:40,857 INFO [PEWorker-1 {}] master.SplitWALManager(105): a23eb99f71f0,44781,1732468442596 WAL count=1, meta=false 2024-11-24T17:14:40,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851}] 2024-11-24T17:14:40,859 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:40,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:41,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38279 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-24T17:14:41,033 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851, size=1.4 K (1434bytes) 2024-11-24T17:14:41,033 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851 2024-11-24T17:14:41,034 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851 after 1ms 2024-11-24T17:14:41,037 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851 took 4ms 2024-11-24T17:14:41,040 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 12eb6cfa7992f8b2d57701a44d5e5526: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T17:14:41,042 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4b5321b4dd571156ebac77ca1fa8b150: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T17:14:41,044 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4a1b996a2c1476427750ae2fbe015bc5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T17:14:41,045 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for e88c3c1672dcd6a9395b368267add589: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T17:14:41,047 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a6d2969d9f925c5b8cb0ff244090dcd4: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T17:14:41,047 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851 so closing down 2024-11-24T17:14:41,047 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T17:14:41,047 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T17:14:41,048 INFO [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 5 edits across 0 Regions in 11 ms; skipped=5; WAL=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851, size=1.4 K, length=1434, corrupted=false, cancelled=false 2024-11-24T17:14:41,048 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851, journal: Splitting hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851, size=1.4 K (1434bytes) at 1732468481033Finishing writing output for hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851 so closing down at 1732468481047 (+14 ms)3 split writer threads finished at 1732468481048 (+1 ms)Processed 5 edits across 0 Regions in 11 ms; skipped=5; WAL=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851, size=1.4 K, length=1434, corrupted=false, cancelled=false at 1732468481048 2024-11-24T17:14:41,048 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851 2024-11-24T17:14:41,048 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a23eb99f71f0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-24T17:14:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40213 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-24T17:14:41,053 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting/a23eb99f71f0%2C44781%2C1732468442596.1732468476851 to hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs 2024-11-24T17:14:41,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-24T17:14:41,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 in 193 msec 2024-11-24T17:14:41,057 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-24T17:14:41,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 in 200 msec 2024-11-24T17:14:41,065 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/WALs/a23eb99f71f0,44781,1732468442596-splitting dir is empty, no logs to split. 2024-11-24T17:14:41,065 INFO [PEWorker-1 {}] master.SplitWALManager(105): a23eb99f71f0,44781,1732468442596 WAL count=0, meta=false 2024-11-24T17:14:41,065 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if a23eb99f71f0,44781,1732468442596 WAL splitting is done? wals=0, meta=false 2024-11-24T17:14:41,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN}] 2024-11-24T17:14:41,073 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:41,074 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:41,076 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:41,077 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:41,079 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:41,080 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:41,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:41,083 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:41,084 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:41,085 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:41,087 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:41,087 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:41,089 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:41,097 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:41,099 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:41,103 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T17:14:41,225 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(204): Hosts are {a23eb99f71f0=0} racks are {/default-rack=0} 2024-11-24T17:14:41,225 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T17:14:41,226 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T17:14:41,226 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T17:14:41,226 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T17:14:41,226 INFO [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T17:14:41,226 INFO [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T17:14:41,226 DEBUG [a23eb99f71f0:40213 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T17:14:41,227 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, regionLocation=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,231 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,232 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,233 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a23eb99f71f0:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:14:41,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1, error=java.net.ConnectException: Call to address=a23eb99f71f0:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T17:14:41,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 from cache 2024-11-24T17:14:41,237 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, regionLocation=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,237 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T17:14:41,237 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34777,1732468442786, seqNum=-1] 2024-11-24T17:14:41,238 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T17:14:41,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34777,1732468442786, seqNum=-1, error=java.net.ConnectException: Call to address=a23eb99f71f0:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34777,1732468442786, seqNum=-1, error=java.net.ConnectException: Call to address=a23eb99f71f0:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34777,1732468442786, seqNum=-1, error=java.net.ConnectException: Call to address=a23eb99f71f0:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34777,1732468442786, seqNum=-1, error=java.net.ConnectException: Call to address=a23eb99f71f0:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34777,1732468442786, seqNum=-1, error=java.net.ConnectException: Call to address=a23eb99f71f0:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address a23eb99f71f0:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,241 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a23eb99f71f0:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:14:41,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,44781,1732468442596, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a23eb99f71f0,34777,1732468442786, seqNum=-1, error=java.net.ConnectException: Call to address=a23eb99f71f0:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address a23eb99f71f0:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a23eb99f71f0/172.17.0.3:44781 2024-11-24T17:14:41,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52975, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T17:14:41,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN because future has completed 2024-11-24T17:14:41,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:41,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN because future has completed 2024-11-24T17:14:41,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:41,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN because future has completed 2024-11-24T17:14:41,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN because future has completed 2024-11-24T17:14:41,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN because future has completed 2024-11-24T17:14:41,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:41,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN because future has completed 2024-11-24T17:14:41,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN because future has completed 2024-11-24T17:14:41,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN because future has completed 2024-11-24T17:14:41,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:41,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:41,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786}] 2024-11-24T17:14:41,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:41,429 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:41,429 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a1b996a2c1476427750ae2fbe015bc5, NAME => 'test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-24T17:14:41,430 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,430 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:41,430 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,430 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12eb6cfa7992f8b2d57701a44d5e5526, server=a23eb99f71f0,38279,1732468442718}] 2024-11-24T17:14:41,456 INFO [StoreOpener-4a1b996a2c1476427750ae2fbe015bc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,464 INFO [StoreOpener-4a1b996a2c1476427750ae2fbe015bc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1b996a2c1476427750ae2fbe015bc5 columnFamilyName family 2024-11-24T17:14:41,464 DEBUG [StoreOpener-4a1b996a2c1476427750ae2fbe015bc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:41,468 INFO [StoreOpener-4a1b996a2c1476427750ae2fbe015bc5-1 {}] regionserver.HStore(327): Store=4a1b996a2c1476427750ae2fbe015bc5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:41,469 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,474 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,480 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,481 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,481 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,497 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,500 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 4a1b996a2c1476427750ae2fbe015bc5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71126390, jitterRate=0.059865802526474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:41,500 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:41,500 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 4a1b996a2c1476427750ae2fbe015bc5: Running coprocessor pre-open hook at 1732468481430Writing region info on filesystem at 1732468481430Initializing all the Stores at 1732468481448 (+18 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468481448Cleaning up temporary data from old regions at 1732468481481 (+33 ms)Running coprocessor post-open hooks at 1732468481500 (+19 ms)Region opened successfully at 1732468481500 2024-11-24T17:14:41,505 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5., pid=75, masterSystemTime=1732468481407 2024-11-24T17:14:41,513 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:41,514 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:41,514 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:41,514 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 3963c202e600d921e58afb7a1966f4f0, NAME => 'test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-24T17:14:41,514 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,515 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:41,515 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,515 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,517 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-24T17:14:41,517 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,519 INFO [StoreOpener-3963c202e600d921e58afb7a1966f4f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,521 INFO [StoreOpener-3963c202e600d921e58afb7a1966f4f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3963c202e600d921e58afb7a1966f4f0 columnFamilyName family 2024-11-24T17:14:41,522 DEBUG [StoreOpener-3963c202e600d921e58afb7a1966f4f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:41,524 INFO [StoreOpener-3963c202e600d921e58afb7a1966f4f0-1 {}] regionserver.HStore(327): Store=3963c202e600d921e58afb7a1966f4f0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:41,524 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,525 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,526 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:41,526 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 12eb6cfa7992f8b2d57701a44d5e5526, NAME => 'test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-24T17:14:41,526 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,526 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:41,526 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,526 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,529 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,529 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,530 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,532 INFO [StoreOpener-12eb6cfa7992f8b2d57701a44d5e5526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,534 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,519 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,536 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 3963c202e600d921e58afb7a1966f4f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71835575, jitterRate=0.07043348252773285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:41,536 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:41,536 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 3963c202e600d921e58afb7a1966f4f0: Running coprocessor pre-open hook at 1732468481516Writing region info on filesystem at 1732468481516Initializing all the Stores at 1732468481518 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468481518Cleaning up temporary data from old regions at 1732468481530 (+12 ms)Running coprocessor post-open hooks at 1732468481536 (+6 ms)Region opened successfully at 1732468481536 2024-11-24T17:14:41,537 INFO [StoreOpener-12eb6cfa7992f8b2d57701a44d5e5526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12eb6cfa7992f8b2d57701a44d5e5526 columnFamilyName family 2024-11-24T17:14:41,537 DEBUG [StoreOpener-12eb6cfa7992f8b2d57701a44d5e5526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:41,537 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0., pid=78, masterSystemTime=1732468481407 2024-11-24T17:14:41,540 INFO [StoreOpener-12eb6cfa7992f8b2d57701a44d5e5526-1 {}] regionserver.HStore(327): Store=12eb6cfa7992f8b2d57701a44d5e5526/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:41,540 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,542 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,545 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:41,545 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,546 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,546 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,550 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,549 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:41,553 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 12eb6cfa7992f8b2d57701a44d5e5526; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61731848, jitterRate=-0.08012378215789795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:41,553 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:41,553 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 12eb6cfa7992f8b2d57701a44d5e5526: Running coprocessor pre-open hook at 1732468481527Writing region info on filesystem at 1732468481527Initializing all the Stores at 1732468481528 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468481528Cleaning up temporary data from old regions at 1732468481546 (+18 ms)Running coprocessor post-open hooks at 1732468481553 (+7 ms)Region opened successfully at 1732468481553 2024-11-24T17:14:41,557 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526., pid=82, masterSystemTime=1732468481518 2024-11-24T17:14:41,562 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:41,562 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:41,562 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:41,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 93d787842f2c5c53adaf00e6e90673e7, NAME => 'test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-24T17:14:41,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,563 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:41,561 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-24T17:14:41,565 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:41,565 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,565 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,565 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12eb6cfa7992f8b2d57701a44d5e5526, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:41,568 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:41,570 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12eb6cfa7992f8b2d57701a44d5e5526, server=a23eb99f71f0,38279,1732468442718 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,570 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 58c2a35cef97d825f565d67ca7006c36, NAME => 'test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-24T17:14:41,571 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,571 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:41,571 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,571 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,575 INFO [StoreOpener-58c2a35cef97d825f565d67ca7006c36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,580 INFO [StoreOpener-93d787842f2c5c53adaf00e6e90673e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,581 INFO [StoreOpener-58c2a35cef97d825f565d67ca7006c36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c2a35cef97d825f565d67ca7006c36 columnFamilyName family 2024-11-24T17:14:41,581 DEBUG [StoreOpener-58c2a35cef97d825f565d67ca7006c36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:41,582 INFO [StoreOpener-58c2a35cef97d825f565d67ca7006c36-1 {}] regionserver.HStore(327): Store=58c2a35cef97d825f565d67ca7006c36/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:41,582 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,583 INFO [StoreOpener-93d787842f2c5c53adaf00e6e90673e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93d787842f2c5c53adaf00e6e90673e7 columnFamilyName family 2024-11-24T17:14:41,583 DEBUG [StoreOpener-93d787842f2c5c53adaf00e6e90673e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:41,587 INFO [StoreOpener-93d787842f2c5c53adaf00e6e90673e7-1 {}] regionserver.HStore(327): Store=93d787842f2c5c53adaf00e6e90673e7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:41,587 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,587 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,590 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,591 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,593 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,593 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,597 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,598 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,598 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,598 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,601 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 58c2a35cef97d825f565d67ca7006c36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61327946, jitterRate=-0.0861423909664154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:41,601 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:41,601 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 58c2a35cef97d825f565d67ca7006c36: Running coprocessor pre-open hook at 1732468481571Writing region info on filesystem at 1732468481571Initializing all the Stores at 1732468481574 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468481574Cleaning up temporary data from old regions at 1732468481593 (+19 ms)Running coprocessor post-open hooks at 1732468481601 (+8 ms)Region opened successfully at 1732468481601 2024-11-24T17:14:41,605 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36., pid=80, masterSystemTime=1732468481560 2024-11-24T17:14:41,605 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,607 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 93d787842f2c5c53adaf00e6e90673e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71236048, jitterRate=0.061499834060668945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:41,607 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:41,607 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 93d787842f2c5c53adaf00e6e90673e7: Running coprocessor pre-open hook at 1732468481565Writing region info on filesystem at 1732468481565Initializing all the Stores at 1732468481569 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468481569Cleaning up temporary data from old regions at 1732468481598 (+29 ms)Running coprocessor post-open hooks at 1732468481607 (+9 ms)Region opened successfully at 1732468481607 2024-11-24T17:14:41,620 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:41,620 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:41,620 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:41,621 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f16dc15c0e0aff7efc028181a4fdbe3, NAME => 'test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-24T17:14:41,621 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,621 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:41,621 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,621 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,623 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7., pid=76, masterSystemTime=1732468481518 2024-11-24T17:14:41,630 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:41,630 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:41,630 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:41,630 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => e88c3c1672dcd6a9395b368267add589, NAME => 'test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-24T17:14:41,631 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,631 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:41,631 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,631 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,632 INFO [StoreOpener-6f16dc15c0e0aff7efc028181a4fdbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,635 INFO [StoreOpener-e88c3c1672dcd6a9395b368267add589-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,637 INFO [StoreOpener-6f16dc15c0e0aff7efc028181a4fdbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f16dc15c0e0aff7efc028181a4fdbe3 columnFamilyName family 2024-11-24T17:14:41,637 DEBUG [StoreOpener-6f16dc15c0e0aff7efc028181a4fdbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:41,637 INFO [StoreOpener-e88c3c1672dcd6a9395b368267add589-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e88c3c1672dcd6a9395b368267add589 columnFamilyName family 2024-11-24T17:14:41,638 DEBUG [StoreOpener-e88c3c1672dcd6a9395b368267add589-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:41,638 INFO [StoreOpener-6f16dc15c0e0aff7efc028181a4fdbe3-1 {}] regionserver.HStore(327): Store=6f16dc15c0e0aff7efc028181a4fdbe3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:41,638 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,641 INFO [StoreOpener-e88c3c1672dcd6a9395b368267add589-1 {}] regionserver.HStore(327): Store=e88c3c1672dcd6a9395b368267add589/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:41,642 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,643 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,649 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,655 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,656 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,656 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,663 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:41,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:41,664 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:41,664 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:41,664 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,665 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,665 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:41,665 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:41,665 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:41,666 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:41,666 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,666 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,666 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened e88c3c1672dcd6a9395b368267add589; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68182113, jitterRate=0.015992656350135803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:41,666 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:41,667 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for e88c3c1672dcd6a9395b368267add589: Running coprocessor pre-open hook at 1732468481631Writing region info on filesystem at 1732468481631Initializing all the Stores at 1732468481635 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468481635Cleaning up temporary data from old regions at 1732468481657 (+22 ms)Running coprocessor post-open hooks at 1732468481667 (+10 ms)Region opened successfully at 1732468481667 2024-11-24T17:14:41,673 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589., pid=81, masterSystemTime=1732468481518 2024-11-24T17:14:41,674 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,676 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 10 msec 2024-11-24T17:14:41,693 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,705 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 6f16dc15c0e0aff7efc028181a4fdbe3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66331415, jitterRate=-0.011584892868995667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:41,705 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:41,705 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 6f16dc15c0e0aff7efc028181a4fdbe3: Running coprocessor pre-open hook at 1732468481622Writing region info on filesystem at 1732468481622Initializing all the Stores at 1732468481623 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468481623Cleaning up temporary data from old regions at 1732468481666 (+43 ms)Running coprocessor post-open hooks at 1732468481705 (+39 ms)Region opened successfully at 1732468481705 2024-11-24T17:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:41,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,712 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3., pid=79, masterSystemTime=1732468481560 2024-11-24T17:14:41,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:41,713 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,714 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "a23eb99f71f0" port: 38279 start_code: 1732468442718 } transition { transition_code: OPENED region_info { region_id: 1732468449655 table_name { namespace: "default" qualifier: "test" } start_key: "eee" end_key: "fff" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 81 initiating_master_active_time: 1732468442958 }; retry (#0) after 1009ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-24T17:14:41,717 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "a23eb99f71f0" port: 34777 start_code: 1732468442786 } transition { transition_code: OPENED region_info { region_id: 1732468449655 table_name { namespace: "default" qualifier: "test" } start_key: "ttt" end_key: "uuu" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732468442958 }; retry (#0) after 1009ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-24T17:14:41,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:41,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:41,732 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:41,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12eb6cfa7992f8b2d57701a44d5e5526, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,739 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,740 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,740 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:41,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:41,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:41,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:41,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:41,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:41,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:41,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:41,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12eb6cfa7992f8b2d57701a44d5e5526, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,747 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 70 msec 2024-11-24T17:14:41,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:41,761 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:41,761 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:41,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:41,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:41,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:41,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:41,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:41,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:41,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:41,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:41,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:41,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:41,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:41,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:41,795 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:41,795 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:41,815 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468511815 2024-11-24T17:14:41,816 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:41,816 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,828 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:41,840 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:41,840 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:41,840 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,38279,1732468442718 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:41,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:41,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:41,904 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:41,904 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468541904 2024-11-24T17:14:41,904 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468601904 2024-11-24T17:14:41,904 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 88 msec 2024-11-24T17:14:41,904 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:41,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:41,904 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:41,904 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:41,908 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:41,908 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:41,908 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:41,914 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:41,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 13 msec 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:41,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:41,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:41,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:41,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:41,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:41,958 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:41,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:41,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,960 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:41,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:41,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:41,964 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure e88c3c1672dcd6a9395b368267add589, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:41,964 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 36 msec 2024-11-24T17:14:41,967 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:41,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:41,968 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:41,969 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:41,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:41,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:41,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:41,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:41,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:41,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:42,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:42,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:42,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:42,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:42,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:42,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:42,008 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:42,036 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468512036 2024-11-24T17:14:42,036 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:42,036 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,038 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:42,049 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:42,049 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,049 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:42,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,067 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:42,067 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468542067 2024-11-24T17:14:42,067 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468602067 2024-11-24T17:14:42,067 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-24T17:14:42,067 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:42,067 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,067 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:42,068 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:42,070 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:42,070 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:42,070 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:42,070 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:42,071 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:42,075 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,078 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,082 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:42,082 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,084 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:42,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:42,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:42,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:42,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,090 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,091 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,091 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,091 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,091 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,091 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,091 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,091 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,091 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,091 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2a35cef97d825f565d67ca7006c36, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,092 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 20 msec 2024-11-24T17:14:42,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:42,102 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:42,102 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:42,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,129 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:42,133 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:42,176 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468512176 2024-11-24T17:14:42,176 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:42,176 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,178 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:42,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:42,189 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,189 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741830_1006 (size=153316) 2024-11-24T17:14:42,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:42,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,213 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:42,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468542213 2024-11-24T17:14:42,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468602213 2024-11-24T17:14:42,213 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-11-24T17:14:42,213 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:42,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,213 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:42,213 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:42,218 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:42,218 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:42,218 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:42,218 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:42,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:42,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:42,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:42,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:42,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:42,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,237 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,237 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,237 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f16dc15c0e0aff7efc028181a4fdbe3, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,238 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 19 msec 2024-11-24T17:14:42,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:42,245 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:42,245 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:42,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,258 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:42,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:42,288 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468512288 2024-11-24T17:14:42,288 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:42,289 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,291 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:42,298 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:42,298 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,298 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:42,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,315 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:42,316 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468542315 2024-11-24T17:14:42,316 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468602316 2024-11-24T17:14:42,316 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-24T17:14:42,316 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:42,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,316 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:42,316 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:42,320 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:42,320 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:42,320 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:42,321 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:42,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 6 msec 2024-11-24T17:14:42,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:42,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:42,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:42,345 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:42,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:42,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:42,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3963c202e600d921e58afb7a1966f4f0, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,351 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 22 msec 2024-11-24T17:14:42,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:42,355 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:42,355 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:42,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,369 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,369 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,384 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:42,384 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:42,385 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:42,385 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:42,385 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:42,385 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:42,385 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:42,385 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:42,408 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468512408 2024-11-24T17:14:42,408 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:42,412 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,414 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:42,418 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:42,418 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,418 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,426 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,427 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:42,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,441 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:42,441 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468542441 2024-11-24T17:14:42,441 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468602441 2024-11-24T17:14:42,441 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-24T17:14:42,441 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:42,441 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,441 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:42,442 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:42,444 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:42,445 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:42,445 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:42,445 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:42,460 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 14 msec 2024-11-24T17:14:42,466 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:42,466 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,466 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,466 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,466 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:42,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:42,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:42,475 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T17:14:42,475 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:42,475 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,475 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,475 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:42,475 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,475 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,500 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,501 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,501 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,501 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,501 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,501 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,501 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,501 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,501 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,501 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b5321b4dd571156ebac77ca1fa8b150, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,502 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 41 msec 2024-11-24T17:14:42,502 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:42,504 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:42,504 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:42,511 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:42,539 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:42,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:42,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:42,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:42,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:42,553 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:42,553 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:42,553 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:42,553 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:42,564 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468512564 2024-11-24T17:14:42,564 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:42,566 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,574 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:42,577 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:42,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:42,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:42,585 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:42,585 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468542585 2024-11-24T17:14:42,585 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468602585 2024-11-24T17:14:42,585 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 20 msec 2024-11-24T17:14:42,592 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:42,592 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,593 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,728 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.TransitRegionStateProcedure(535): There is no outstanding remote region procedure for state=OPENING, location=a23eb99f71f0,38279,1732468442718, table=test, region=e88c3c1672dcd6a9395b368267add589, serverName=a23eb99f71f0,38279,1732468442718, code=OPENED, seqId=2, proc=pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN, should be a retry, ignore 2024-11-24T17:14:42,729 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] assignment.TransitRegionStateProcedure(535): There is no outstanding remote region procedure for state=OPENING, location=a23eb99f71f0,34777,1732468442786, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, serverName=a23eb99f71f0,34777,1732468442786, code=OPENED, seqId=2, proc=pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN, should be a retry, ignore 2024-11-24T17:14:42,730 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "a23eb99f71f0" port: 38279 start_code: 1732468442718 } transition { transition_code: OPENED region_info { region_id: 1732468449655 table_name { namespace: "default" qualifier: "test" } start_key: "eee" end_key: "fff" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 81 initiating_master_active_time: 1732468442958 } 2024-11-24T17:14:42,730 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:42,730 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:42,731 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. 2024-11-24T17:14:42,731 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b5321b4dd571156ebac77ca1fa8b150, NAME => 'test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-24T17:14:42,731 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,732 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T17:14:42,732 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,739 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "a23eb99f71f0" port: 34777 start_code: 1732468442786 } transition { transition_code: OPENED region_info { region_id: 1732468449655 table_name { namespace: "default" qualifier: "test" } start_key: "ttt" end_key: "uuu" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732468442958 } 2024-11-24T17:14:42,739 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:42,739 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:42,732 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,769 INFO [StoreOpener-4b5321b4dd571156ebac77ca1fa8b150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,776 INFO [StoreOpener-4b5321b4dd571156ebac77ca1fa8b150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5321b4dd571156ebac77ca1fa8b150 columnFamilyName family 2024-11-24T17:14:42,776 DEBUG [StoreOpener-4b5321b4dd571156ebac77ca1fa8b150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T17:14:42,783 INFO [StoreOpener-4b5321b4dd571156ebac77ca1fa8b150-1 {}] regionserver.HStore(327): Store=4b5321b4dd571156ebac77ca1fa8b150/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T17:14:42,783 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,796 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,802 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,809 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,809 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,828 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,835 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:42,835 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:42,835 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:42,836 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:42,836 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:42,836 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:42,836 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:42,837 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 4b5321b4dd571156ebac77ca1fa8b150; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67749298, jitterRate=0.00954321026802063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T17:14:42,837 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b5321b4dd571156ebac77ca1fa8b150 2024-11-24T17:14:42,837 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:42,837 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 4b5321b4dd571156ebac77ca1fa8b150: Running coprocessor pre-open hook at 1732468482744Writing region info on filesystem at 1732468482744Initializing all the Stores at 1732468482760 (+16 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732468482760Cleaning up temporary data from old regions at 1732468482809 (+49 ms)Running coprocessor post-open hooks at 1732468482837 (+28 ms)Region opened successfully at 1732468482837 2024-11-24T17:14:42,841 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-24T17:14:42,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:42,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:42,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:42,852 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:42,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:42,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:42,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:42,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,856 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,856 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,857 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:42,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,857 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93d787842f2c5c53adaf00e6e90673e7, server=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:42,858 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-24T17:14:42,858 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150., pid=77, masterSystemTime=1732468481518 2024-11-24T17:14:42,859 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,862 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "a23eb99f71f0" port: 38279 start_code: 1732468442718 } transition { transition_code: OPENED region_info { region_id: 1732468449655 table_name { namespace: "default" qualifier: "test" } start_key: "ccc" end_key: "ddd" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1732468442958 }; retry (#0) after 1006ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-24T17:14:42,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:42,868 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:42,869 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:42,967 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:42,969 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,059 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:43,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:43,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:43,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:43,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:43,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:43,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:43,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:43,098 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468513098 2024-11-24T17:14:43,098 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:43,100 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,102 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:43,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,116 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:43,118 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:43,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,118 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:43,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,122 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:43,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468543122 2024-11-24T17:14:43,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468603122 2024-11-24T17:14:43,122 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-24T17:14:43,122 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:43,122 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,122 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:43,122 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:43,123 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:43,123 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:43,123 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:43,123 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:43,124 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T17:14:43,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,128 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:43,129 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:43,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:43,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:43,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:43,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:43,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:43,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:43,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:43,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,139 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-24T17:14:43,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:43,141 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:43,141 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:43,162 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,162 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,483 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,483 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:43,483 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:43,483 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:43,483 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:43,483 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:43,483 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:43,484 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:43,484 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:43,488 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468513488 2024-11-24T17:14:43,488 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:43,488 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,490 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:43,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,498 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:43,498 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:43,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,499 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:43,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,501 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:43,501 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468543501 2024-11-24T17:14:43,501 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468603501 2024-11-24T17:14:43,501 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T17:14:43,501 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:43,501 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,501 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:43,501 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:43,502 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:43,502 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:43,502 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:43,503 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:43,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:43,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:43,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:43,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,511 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,511 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,511 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:43,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN 2024-11-24T17:14:43,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-24T17:14:43,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:43,513 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:43,514 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:43,584 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,585 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:43,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:43,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:43,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:43,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:43,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:43,669 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:43,669 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=e88c3c1672dcd6a9395b368267add589, ASSIGN to state=OFFLINE, location=null, table=test, region=e88c3c1672dcd6a9395b368267add589 to restore RIT 2024-11-24T17:14:43,679 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468513679 2024-11-24T17:14:43,679 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:43,681 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,682 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:43,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,692 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:43,692 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:43,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,692 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:43,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,698 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:43,698 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468543698 2024-11-24T17:14:43,698 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468603698 2024-11-24T17:14:43,698 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-24T17:14:43,698 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:43,698 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,698 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:43,700 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:43,704 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:43,704 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:43,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:43,705 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:43,712 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 7 msec 2024-11-24T17:14:43,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:43,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:43,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:43,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:43,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,722 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,722 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,722 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,723 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,723 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,723 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,723 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,723 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,723 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,723 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,723 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,723 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN 2024-11-24T17:14:43,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-24T17:14:43,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:43,724 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:43,725 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:43,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,766 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:43,766 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:43,766 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:43,766 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:43,766 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:43,766 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:43,766 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b5321b4dd571156ebac77ca1fa8b150, ASSIGN to state=OFFLINE, location=null, table=test, region=4b5321b4dd571156ebac77ca1fa8b150 to restore RIT 2024-11-24T17:14:43,769 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468513769 2024-11-24T17:14:43,769 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:43,770 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,771 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:43,777 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:43,777 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,777 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:43,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,785 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:43,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468543785 2024-11-24T17:14:43,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468603785 2024-11-24T17:14:43,785 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-24T17:14:43,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:43,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,785 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:43,785 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:43,786 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:43,786 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:43,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:43,786 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:43,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-24T17:14:43,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:43,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,798 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,798 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,798 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,798 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,798 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:43,798 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:43,799 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:43,799 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,801 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,802 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,802 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,802 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,802 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,802 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,802 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,802 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN 2024-11-24T17:14:43,802 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-24T17:14:43,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:43,805 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:43,805 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:43,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:43,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:43,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:43,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:43,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:43,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=58c2a35cef97d825f565d67ca7006c36, ASSIGN to state=OFFLINE, location=null, table=test, region=58c2a35cef97d825f565d67ca7006c36 to restore RIT 2024-11-24T17:14:43,831 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468513831 2024-11-24T17:14:43,831 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:43,831 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,835 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:43,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:43,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,841 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:43,841 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,842 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,842 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:43,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468543842 2024-11-24T17:14:43,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468603842 2024-11-24T17:14:43,842 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-24T17:14:43,843 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:43,843 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,843 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:43,843 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:43,843 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:43,843 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:43,843 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:43,844 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:43,852 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 8 msec 2024-11-24T17:14:43,873 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,877 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:43,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:43,878 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "a23eb99f71f0" port: 38279 start_code: 1732468442718 } transition { transition_code: OPENED region_info { region_id: 1732468449655 table_name { namespace: "default" qualifier: "test" } start_key: "ccc" end_key: "ddd" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1732468442958 }; retry (#1) after 2004ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor61.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-24T17:14:43,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:43,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:43,880 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,882 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,882 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,882 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,882 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,882 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,882 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,882 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,882 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,882 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN 2024-11-24T17:14:43,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 29 msec 2024-11-24T17:14:43,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:43,884 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:43,884 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:43,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:43,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:43,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:43,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:43,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,904 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3, ASSIGN to state=OFFLINE, location=null, table=test, region=6f16dc15c0e0aff7efc028181a4fdbe3 to restore RIT 2024-11-24T17:14:43,912 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468513912 2024-11-24T17:14:43,912 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:43,912 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,914 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:43,920 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:43,920 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,921 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:43,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:43,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:43,925 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:43,925 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468543925 2024-11-24T17:14:43,925 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468603925 2024-11-24T17:14:43,925 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T17:14:43,925 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:43,925 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:43,925 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:43,925 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:43,925 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:43,925 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:43,926 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:43,926 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:43,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-24T17:14:43,931 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:43,931 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:43,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:43,933 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:43,933 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:43,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,936 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:43,936 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,936 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:43,936 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,936 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:43,936 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,936 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:43,936 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:43,936 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN 2024-11-24T17:14:43,936 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T17:14:43,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:43,945 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:43,945 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:43,980 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,995 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:43,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:43,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:43,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:44,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:44,001 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:44,001 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93d787842f2c5c53adaf00e6e90673e7, ASSIGN to state=OFFLINE, location=null, table=test, region=93d787842f2c5c53adaf00e6e90673e7 to restore RIT 2024-11-24T17:14:44,012 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514012 2024-11-24T17:14:44,012 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,014 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,016 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,032 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,032 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:44,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,032 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,034 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544034 2024-11-24T17:14:44,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604034 2024-11-24T17:14:44,034 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-11-24T17:14:44,034 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,034 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,034 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,036 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,036 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,036 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,037 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,043 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 6 msec 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:44,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:44,049 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,050 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,050 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:44,050 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,050 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN 2024-11-24T17:14:44,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T17:14:44,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,051 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,051 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,078 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:44,079 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:44,079 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3963c202e600d921e58afb7a1966f4f0, ASSIGN to state=OFFLINE, location=null, table=test, region=3963c202e600d921e58afb7a1966f4f0 to restore RIT 2024-11-24T17:14:44,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514084 2024-11-24T17:14:44,084 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,088 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,091 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,091 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:44,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,091 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,098 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,098 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544098 2024-11-24T17:14:44,098 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604098 2024-11-24T17:14:44,098 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-24T17:14:44,098 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,098 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,099 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,099 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,099 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,099 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,099 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,100 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:44,102 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,105 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:44,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,107 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:44,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,107 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN 2024-11-24T17:14:44,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T17:14:44,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,109 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,109 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,205 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,206 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,224 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,224 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,224 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:44,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4a1b996a2c1476427750ae2fbe015bc5, ASSIGN to state=OFFLINE, location=null, table=test, region=4a1b996a2c1476427750ae2fbe015bc5 to restore RIT 2024-11-24T17:14:44,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514225 2024-11-24T17:14:44,226 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,226 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,230 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,234 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:44,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,234 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,238 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544238 2024-11-24T17:14:44,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604238 2024-11-24T17:14:44,238 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T17:14:44,238 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,238 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,238 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,238 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,240 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,240 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,240 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,240 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,241 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T17:14:44,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:44,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:44,247 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,248 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,248 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,248 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN 2024-11-24T17:14:44,248 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T17:14:44,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,250 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,250 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12eb6cfa7992f8b2d57701a44d5e5526, ASSIGN to state=OFFLINE, location=null, table=test, region=12eb6cfa7992f8b2d57701a44d5e5526 to restore RIT 2024-11-24T17:14:44,276 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514276 2024-11-24T17:14:44,276 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,276 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,278 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,281 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,281 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,281 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,284 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,284 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544284 2024-11-24T17:14:44,284 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604284 2024-11-24T17:14:44,284 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T17:14:44,284 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,284 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,284 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,284 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,285 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,285 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,285 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,285 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,286 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:44,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:44,288 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,290 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T17:14:44,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,291 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,292 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,326 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,326 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,352 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514352 2024-11-24T17:14:44,353 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,353 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,355 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,357 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,357 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,357 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,360 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,360 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544360 2024-11-24T17:14:44,360 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604360 2024-11-24T17:14:44,361 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T17:14:44,361 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,361 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,361 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,361 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,361 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,361 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,361 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,361 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,362 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:44,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.1732468476851, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,366 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T17:14:44,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,367 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,367 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,431 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514431 2024-11-24T17:14:44,431 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,431 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,433 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,440 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,440 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,440 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,446 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,446 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544446 2024-11-24T17:14:44,446 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604446 2024-11-24T17:14:44,446 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-24T17:14:44,446 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,446 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,446 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,448 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,448 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,448 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,448 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,449 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:44,453 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,456 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,460 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,461 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,461 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,461 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T17:14:44,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:44,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:44,463 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,463 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,463 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,463 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,463 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a23eb99f71f0,34777,1732468442786 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,463 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-24T17:14:44,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,464 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,464 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,516 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514516 2024-11-24T17:14:44,516 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,520 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,522 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,529 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:44,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,533 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,533 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544533 2024-11-24T17:14:44,533 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604533 2024-11-24T17:14:44,533 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 17 msec 2024-11-24T17:14:44,533 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,533 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,534 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,534 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,536 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,536 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,537 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,537 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,544 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 6 msec 2024-11-24T17:14:44,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T17:14:44,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,547 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T17:14:44,548 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,548 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,548 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,548 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,548 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T17:14:44,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,557 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,557 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,560 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,561 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,681 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514681 2024-11-24T17:14:44,681 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,683 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,691 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,691 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,691 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,692 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,692 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544692 2024-11-24T17:14:44,692 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604692 2024-11-24T17:14:44,692 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-24T17:14:44,693 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,693 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,693 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,693 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,693 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,694 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:44,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T17:14:44,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,697 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,697 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,698 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,698 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,698 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,698 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T17:14:44,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,699 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,699 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,736 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,740 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,740 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,747 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514747 2024-11-24T17:14:44,747 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,747 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,749 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,753 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,753 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,753 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,762 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,762 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544762 2024-11-24T17:14:44,762 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604762 2024-11-24T17:14:44,763 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-24T17:14:44,763 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:44,763 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,763 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:44,763 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:44,768 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:44,768 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:44,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:44,769 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:44,771 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-24T17:14:44,773 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:44,773 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T17:14:44,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:44,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:44,775 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:44,775 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,775 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:44,775 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:44,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure a23eb99f71f0%2C44781%2C1732468442596.meta.1732468448056.meta, worker=a23eb99f71f0,38279,1732468442718 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T17:14:44,776 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T17:14:44,776 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:44,778 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:44,778 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:44,781 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:44,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:44,825 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468514825 2024-11-24T17:14:44,825 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:44,828 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:44,830 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:44,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:44,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:44,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:44,843 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:44,843 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468544843 2024-11-24T17:14:44,843 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468604843 2024-11-24T17:14:44,844 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-24T17:14:44,848 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T17:14:44,848 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:44,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:44,849 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:45,094 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T17:14:45,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:45,094 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:45,094 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:45,094 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T17:14:45,095 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T17:14:45,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T17:14:45,095 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T17:14:45,096 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T17:14:45,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T17:14:45,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T17:14:45,098 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:45,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T17:14:45,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a6d2969d9f925c5b8cb0ff244090dcd4, REOPEN/MOVE 2024-11-24T17:14:45,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true 2024-11-24T17:14:45,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T17:14:45,100 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T17:14:45,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=65 2024-11-24T17:14:45,104 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a23eb99f71f0,34777,1732468442786, table=hbase:meta, region=1588230740 2024-11-24T17:14:45,104 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a23eb99f71f0,34777,1732468442786, state=OPEN 2024-11-24T17:14:45,113 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40213 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:45,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:45,157 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:45,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:45,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T17:14:45,157 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:45,157 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T17:14:45,168 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732468515168 2024-11-24T17:14:45,168 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T17:14:45,169 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T17:14:45,170 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T17:14:45,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4a1b996a2c1476427750ae2fbe015bc5, server=a23eb99f71f0,34777,1732468442786:java.lang.RuntimeException: inject code bug; ServerCrashProcedure a23eb99f71f0,44781,1732468442596, splitWal=true, meta=true exec-time=5.4610 sec 2024-11-24T17:14:45,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d888fda423b365786e1d269df49d1cfd, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94439407a643ae820e9015188cf23ed6, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b5321b4dd571156ebac77ca1fa8b150, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a1b996a2c1476427750ae2fbe015bc5, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e88c3c1672dcd6a9395b368267add589, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=448f6154a795fbeb01b36798bcbacb8f, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4707e344fe9a8cd93af441dfe2b9492e, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89f671091abaef7d04c744da188c0bf9, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3963c202e600d921e58afb7a1966f4f0, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3132e986d91cd07c924412b1164b0ada, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acf1021fd9104e08951bd81c1946b074, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a61743f7cd3883457b967d55f98541e1, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12eb6cfa7992f8b2d57701a44d5e5526, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1ebd571dd5d321046e5067f5b19ad97, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=68cc51974c64ffdbd8bb6b53a87450da, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bf176f82dd506ad26a9f1d6ebd6f33a, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6d2969d9f925c5b8cb0ff244090dcd4, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=5 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7ba4002d0b6663684be2c126a81b2c74, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4ed93b4907aeb35b406e6de08e103616, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f16dc15c0e0aff7efc028181a4fdbe3, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93d787842f2c5c53adaf00e6e90673e7, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71e89563154e9bf392a9193da474463d, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67115580ab5444880122d14f983db95c, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58c2a35cef97d825f565d67ca7006c36, regionState=OPENING, lastHost=a23eb99f71f0,44781,1732468442596, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=61f230c444c764032b318ed0a81000ab, regionState=OPEN, lastHost=a23eb99f71f0,34777,1732468442786, regionLocation=a23eb99f71f0,34777,1732468442786, openSeqNum=2 2024-11-24T17:14:45,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f35eb03f4e248bda613971e436befd3, regionState=OPEN, lastHost=a23eb99f71f0,38279,1732468442718, regionLocation=a23eb99f71f0,38279,1732468442718, openSeqNum=2 2024-11-24T17:14:45,182 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T17:14:45,182 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732468545182 2024-11-24T17:14:45,182 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732468605182 2024-11-24T17:14:45,182 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-24T17:14:45,212 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=352 (was 389), OpenFileDescriptor=602 (was 629), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1226 (was 1115) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4812 (was 5164) 2024-11-24T17:14:45,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T17:14:45,212 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T17:14:45,216 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:14:45,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:45,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:45,219 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T17:14:45,219 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1670137913, stopped=false 2024-11-24T17:14:45,219 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a23eb99f71f0,40213,1732468441272 2024-11-24T17:14:45,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T17:14:45,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T17:14:45,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:45,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:45,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T17:14:45,259 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T17:14:45,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:45,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T17:14:45,259 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:14:45,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:45,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:45,260 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a23eb99f71f0,38279,1732468442718' ***** 2024-11-24T17:14:45,260 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T17:14:45,260 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a23eb99f71f0,34777,1732468442786' ***** 2024-11-24T17:14:45,260 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T17:14:45,260 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T17:14:45,260 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T17:14:45,260 INFO [RS:1;a23eb99f71f0:38279 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T17:14:45,262 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:14:45,263 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:14:45,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T17:14:45,263 INFO [RS:1;a23eb99f71f0:38279 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T17:14:45,272 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 4f35eb03f4e248bda613971e436befd3 2024-11-24T17:14:45,276 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T17:14:45,276 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 4ed93b4907aeb35b406e6de08e103616 2024-11-24T17:14:45,277 INFO [RS:2;a23eb99f71f0:34777 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for b1ebd571dd5d321046e5067f5b19ad97 2024-11-24T17:14:45,277 INFO [RS:2;a23eb99f71f0:34777 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 94439407a643ae820e9015188cf23ed6 2024-11-24T17:14:45,277 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 4707e344fe9a8cd93af441dfe2b9492e 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 448f6154a795fbeb01b36798bcbacb8f 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 67115580ab5444880122d14f983db95c 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 89f671091abaef7d04c744da188c0bf9 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 3bf176f82dd506ad26a9f1d6ebd6f33a 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for a61743f7cd3883457b967d55f98541e1 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:45,277 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 12eb6cfa7992f8b2d57701a44d5e5526 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(3091): Received CLOSE for 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(959): stopping server a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T17:14:45,277 INFO [RS:1;a23eb99f71f0:38279 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a23eb99f71f0:38279. 2024-11-24T17:14:45,278 DEBUG [RS:1;a23eb99f71f0:38279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:14:45,278 DEBUG [RS:1;a23eb99f71f0:38279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:45,278 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-24T17:14:45,278 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 58c2a35cef97d825f565d67ca7006c36 2024-11-24T17:14:45,278 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1351): Waiting on 12eb6cfa7992f8b2d57701a44d5e5526, 3bf176f82dd506ad26a9f1d6ebd6f33a, 448f6154a795fbeb01b36798bcbacb8f, 4b5321b4dd571156ebac77ca1fa8b150, 4ed93b4907aeb35b406e6de08e103616, 4f35eb03f4e248bda613971e436befd3, 67115580ab5444880122d14f983db95c, 89f671091abaef7d04c744da188c0bf9, 93d787842f2c5c53adaf00e6e90673e7, 94439407a643ae820e9015188cf23ed6, a61743f7cd3883457b967d55f98541e1, b1ebd571dd5d321046e5067f5b19ad97, e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for acf1021fd9104e08951bd81c1946b074 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 61f230c444c764032b318ed0a81000ab 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 71e89563154e9bf392a9193da474463d 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 6f16dc15c0e0aff7efc028181a4fdbe3 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 3963c202e600d921e58afb7a1966f4f0 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 68cc51974c64ffdbd8bb6b53a87450da 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 4a1b996a2c1476427750ae2fbe015bc5 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 7ba4002d0b6663684be2c126a81b2c74 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(959): stopping server a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T17:14:45,279 INFO [RS:2;a23eb99f71f0:34777 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a23eb99f71f0:34777. 2024-11-24T17:14:45,279 DEBUG [RS:2;a23eb99f71f0:34777 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T17:14:45,279 DEBUG [RS:2;a23eb99f71f0:34777 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:45,280 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T17:14:45,280 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T17:14:45,280 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T17:14:45,280 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T17:14:45,280 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4f35eb03f4e248bda613971e436befd3, disabling compactions & flushes 2024-11-24T17:14:45,280 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:45,280 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:45,280 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. after waiting 0 ms 2024-11-24T17:14:45,280 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:45,283 INFO [regionserver/a23eb99f71f0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T17:14:45,284 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4707e344fe9a8cd93af441dfe2b9492e, disabling compactions & flushes 2024-11-24T17:14:45,284 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:45,284 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:45,284 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. after waiting 0 ms 2024-11-24T17:14:45,285 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-24T17:14:45,285 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:45,285 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3132e986d91cd07c924412b1164b0ada, 3963c202e600d921e58afb7a1966f4f0, 4707e344fe9a8cd93af441dfe2b9492e, 4a1b996a2c1476427750ae2fbe015bc5, 58c2a35cef97d825f565d67ca7006c36, 61f230c444c764032b318ed0a81000ab, 68cc51974c64ffdbd8bb6b53a87450da, 6f16dc15c0e0aff7efc028181a4fdbe3, 71e89563154e9bf392a9193da474463d, 7ba4002d0b6663684be2c126a81b2c74, a6d2969d9f925c5b8cb0ff244090dcd4, acf1021fd9104e08951bd81c1946b074, d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:45,285 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T17:14:45,285 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T17:14:45,285 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T17:14:45,285 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T17:14:45,285 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T17:14:45,285 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-24T17:14:45,333 INFO [regionserver/a23eb99f71f0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T17:14:45,336 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4707e344fe9a8cd93af441dfe2b9492e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,340 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:45,340 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4707e344fe9a8cd93af441dfe2b9492e: Waiting for close lock at 1732468485284Running coprocessor pre-close hooks at 1732468485284Disabling compacts and flushes for region at 1732468485284Disabling writes for close at 1732468485285 (+1 ms)Writing region close event to WAL at 1732468485292 (+7 ms)Running coprocessor post-close hooks at 1732468485340 (+48 ms)Closed at 1732468485340 2024-11-24T17:14:45,341 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732468449655.4707e344fe9a8cd93af441dfe2b9492e. 2024-11-24T17:14:45,342 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 58c2a35cef97d825f565d67ca7006c36, disabling compactions & flushes 2024-11-24T17:14:45,342 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:45,342 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:45,342 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. after waiting 0 ms 2024-11-24T17:14:45,342 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:45,342 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/info/9208a1c501ac4c0198f73edba5e18aac is 138, key is test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150./info:regioninfo/1732468481229/Put/seqid=0 2024-11-24T17:14:45,356 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4f35eb03f4e248bda613971e436befd3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,363 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:45,363 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4f35eb03f4e248bda613971e436befd3: Waiting for close lock at 1732468485279Running coprocessor pre-close hooks at 1732468485280 (+1 ms)Disabling compacts and flushes for region at 1732468485280Disabling writes for close at 1732468485280Writing region close event to WAL at 1732468485307 (+27 ms)Running coprocessor post-close hooks at 1732468485362 (+55 ms)Closed at 1732468485362 2024-11-24T17:14:45,363 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732468449655.4f35eb03f4e248bda613971e436befd3. 2024-11-24T17:14:45,363 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4ed93b4907aeb35b406e6de08e103616, disabling compactions & flushes 2024-11-24T17:14:45,363 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:45,363 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:45,363 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. after waiting 0 ms 2024-11-24T17:14:45,363 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:45,390 INFO [regionserver/a23eb99f71f0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T17:14:45,391 INFO [regionserver/a23eb99f71f0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T17:14:45,408 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/58c2a35cef97d825f565d67ca7006c36/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,418 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:45,418 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 58c2a35cef97d825f565d67ca7006c36: Waiting for close lock at 1732468485341Running coprocessor pre-close hooks at 1732468485341Disabling compacts and flushes for region at 1732468485341Disabling writes for close at 1732468485342 (+1 ms)Writing region close event to WAL at 1732468485368 (+26 ms)Running coprocessor post-close hooks at 1732468485418 (+50 ms)Closed at 1732468485418 2024-11-24T17:14:45,419 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732468449655.58c2a35cef97d825f565d67ca7006c36. 2024-11-24T17:14:45,419 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing acf1021fd9104e08951bd81c1946b074, disabling compactions & flushes 2024-11-24T17:14:45,419 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:45,419 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:45,419 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. after waiting 0 ms 2024-11-24T17:14:45,419 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:45,436 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4ed93b4907aeb35b406e6de08e103616/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,447 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:45,447 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4ed93b4907aeb35b406e6de08e103616: Waiting for close lock at 1732468485363Running coprocessor pre-close hooks at 1732468485363Disabling compacts and flushes for region at 1732468485363Disabling writes for close at 1732468485363Writing region close event to WAL at 1732468485374 (+11 ms)Running coprocessor post-close hooks at 1732468485447 (+73 ms)Closed at 1732468485447 2024-11-24T17:14:45,447 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732468449655.4ed93b4907aeb35b406e6de08e103616. 2024-11-24T17:14:45,447 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b1ebd571dd5d321046e5067f5b19ad97, disabling compactions & flushes 2024-11-24T17:14:45,447 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:45,447 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:45,448 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. after waiting 0 ms 2024-11-24T17:14:45,448 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:45,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741872_1048 (size=8195) 2024-11-24T17:14:45,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741872_1048 (size=8195) 2024-11-24T17:14:45,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741872_1048 (size=8195) 2024-11-24T17:14:45,473 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/info/9208a1c501ac4c0198f73edba5e18aac 2024-11-24T17:14:45,480 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1351): Waiting on 12eb6cfa7992f8b2d57701a44d5e5526, 3bf176f82dd506ad26a9f1d6ebd6f33a, 448f6154a795fbeb01b36798bcbacb8f, 4b5321b4dd571156ebac77ca1fa8b150, 67115580ab5444880122d14f983db95c, 89f671091abaef7d04c744da188c0bf9, 93d787842f2c5c53adaf00e6e90673e7, 94439407a643ae820e9015188cf23ed6, a61743f7cd3883457b967d55f98541e1, b1ebd571dd5d321046e5067f5b19ad97, e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:45,485 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3132e986d91cd07c924412b1164b0ada, 3963c202e600d921e58afb7a1966f4f0, 4a1b996a2c1476427750ae2fbe015bc5, 61f230c444c764032b318ed0a81000ab, 68cc51974c64ffdbd8bb6b53a87450da, 6f16dc15c0e0aff7efc028181a4fdbe3, 71e89563154e9bf392a9193da474463d, 7ba4002d0b6663684be2c126a81b2c74, a6d2969d9f925c5b8cb0ff244090dcd4, acf1021fd9104e08951bd81c1946b074, d888fda423b365786e1d269df49d1cfd 2024-11-24T17:14:45,488 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/acf1021fd9104e08951bd81c1946b074/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,488 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/b1ebd571dd5d321046e5067f5b19ad97/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,495 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:45,495 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for acf1021fd9104e08951bd81c1946b074: Waiting for close lock at 1732468485419Running coprocessor pre-close hooks at 1732468485419Disabling compacts and flushes for region at 1732468485419Disabling writes for close at 1732468485419Writing region close event to WAL at 1732468485428 (+9 ms)Running coprocessor post-close hooks at 1732468485495 (+67 ms)Closed at 1732468485495 2024-11-24T17:14:45,496 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732468449655.acf1021fd9104e08951bd81c1946b074. 2024-11-24T17:14:45,496 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d888fda423b365786e1d269df49d1cfd, disabling compactions & flushes 2024-11-24T17:14:45,496 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:45,496 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/.tmp/info/9208a1c501ac4c0198f73edba5e18aac as hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/info/9208a1c501ac4c0198f73edba5e18aac 2024-11-24T17:14:45,496 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:45,496 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b1ebd571dd5d321046e5067f5b19ad97: Waiting for close lock at 1732468485447Running coprocessor pre-close hooks at 1732468485447Disabling compacts and flushes for region at 1732468485447Disabling writes for close at 1732468485448 (+1 ms)Writing region close event to WAL at 1732468485460 (+12 ms)Running coprocessor post-close hooks at 1732468485496 (+36 ms)Closed at 1732468485496 2024-11-24T17:14:45,496 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732468449655.b1ebd571dd5d321046e5067f5b19ad97. 2024-11-24T17:14:45,496 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 94439407a643ae820e9015188cf23ed6, disabling compactions & flushes 2024-11-24T17:14:45,497 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:45,497 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:45,497 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. after waiting 0 ms 2024-11-24T17:14:45,497 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:45,496 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:45,500 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732468449655.d888fda423b365786e1d269df49d1cfd. after waiting 0 ms 2024-11-24T17:14:45,500 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:45,528 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/d888fda423b365786e1d269df49d1cfd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,529 INFO [regionserver/a23eb99f71f0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T17:14:45,529 INFO [regionserver/a23eb99f71f0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T17:14:45,530 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:45,530 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d888fda423b365786e1d269df49d1cfd: Waiting for close lock at 1732468485496Running coprocessor pre-close hooks at 1732468485496Disabling compacts and flushes for region at 1732468485496Disabling writes for close at 1732468485500 (+4 ms)Writing region close event to WAL at 1732468485504 (+4 ms)Running coprocessor post-close hooks at 1732468485529 (+25 ms)Closed at 1732468485530 (+1 ms) 2024-11-24T17:14:45,530 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732468449655.d888fda423b365786e1d269df49d1cfd. 2024-11-24T17:14:45,530 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 61f230c444c764032b318ed0a81000ab, disabling compactions & flushes 2024-11-24T17:14:45,530 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:45,530 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:45,530 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. after waiting 0 ms 2024-11-24T17:14:45,530 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:45,532 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/info/9208a1c501ac4c0198f73edba5e18aac, entries=24, sequenceid=75, filesize=8.0 K 2024-11-24T17:14:45,532 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/94439407a643ae820e9015188cf23ed6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,537 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:45,537 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 94439407a643ae820e9015188cf23ed6: Waiting for close lock at 1732468485496Running coprocessor pre-close hooks at 1732468485496Disabling compacts and flushes for region at 1732468485496Disabling writes for close at 1732468485497 (+1 ms)Writing region close event to WAL at 1732468485498 (+1 ms)Running coprocessor post-close hooks at 1732468485537 (+39 ms)Closed at 1732468485537 2024-11-24T17:14:45,537 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732468449655.94439407a643ae820e9015188cf23ed6. 2024-11-24T17:14:45,538 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 448f6154a795fbeb01b36798bcbacb8f, disabling compactions & flushes 2024-11-24T17:14:45,538 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:45,538 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:45,538 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. after waiting 0 ms 2024-11-24T17:14:45,538 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:45,539 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 253ms, sequenceid=75, compaction requested=false 2024-11-24T17:14:45,573 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/61f230c444c764032b318ed0a81000ab/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,577 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:45,577 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 61f230c444c764032b318ed0a81000ab: Waiting for close lock at 1732468485530Running coprocessor pre-close hooks at 1732468485530Disabling compacts and flushes for region at 1732468485530Disabling writes for close at 1732468485530Writing region close event to WAL at 1732468485534 (+4 ms)Running coprocessor post-close hooks at 1732468485577 (+43 ms)Closed at 1732468485577 2024-11-24T17:14:45,577 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/448f6154a795fbeb01b36798bcbacb8f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,577 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732468449655.61f230c444c764032b318ed0a81000ab. 2024-11-24T17:14:45,577 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 71e89563154e9bf392a9193da474463d, disabling compactions & flushes 2024-11-24T17:14:45,577 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:45,577 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:45,577 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. after waiting 0 ms 2024-11-24T17:14:45,578 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:45,583 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:45,583 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 448f6154a795fbeb01b36798bcbacb8f: Waiting for close lock at 1732468485538Running coprocessor pre-close hooks at 1732468485538Disabling compacts and flushes for region at 1732468485538Disabling writes for close at 1732468485538Writing region close event to WAL at 1732468485539 (+1 ms)Running coprocessor post-close hooks at 1732468485582 (+43 ms)Closed at 1732468485583 (+1 ms) 2024-11-24T17:14:45,583 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732468449655.448f6154a795fbeb01b36798bcbacb8f. 2024-11-24T17:14:45,583 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 67115580ab5444880122d14f983db95c, disabling compactions & flushes 2024-11-24T17:14:45,583 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:45,583 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:45,583 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732468449655.67115580ab5444880122d14f983db95c. after waiting 0 ms 2024-11-24T17:14:45,583 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:45,586 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-24T17:14:45,587 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T17:14:45,587 INFO [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T17:14:45,587 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732468485285Running coprocessor pre-close hooks at 1732468485285Disabling compacts and flushes for region at 1732468485285Disabling writes for close at 1732468485285Obtaining lock to block concurrent updates at 1732468485285Preparing flush snapshotting stores in 1588230740 at 1732468485285Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732468485291 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732468485304 (+13 ms)Flushing 1588230740/info: creating writer at 1732468485304Flushing 1588230740/info: appending metadata at 1732468485342 (+38 ms)Flushing 1588230740/info: closing flushed file at 1732468485342Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c475a02: reopening flushed file at 1732468485489 (+147 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 253ms, sequenceid=75, compaction requested=false at 1732468485539 (+50 ms)Writing region close event to WAL at 1732468485548 (+9 ms)Running coprocessor post-close hooks at 1732468485587 (+39 ms)Closed at 1732468485587 2024-11-24T17:14:45,588 DEBUG [RS_CLOSE_META-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T17:14:45,632 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/67115580ab5444880122d14f983db95c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,633 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/71e89563154e9bf392a9193da474463d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,634 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:45,634 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 67115580ab5444880122d14f983db95c: Waiting for close lock at 1732468485583Running coprocessor pre-close hooks at 1732468485583Disabling compacts and flushes for region at 1732468485583Disabling writes for close at 1732468485583Writing region close event to WAL at 1732468485586 (+3 ms)Running coprocessor post-close hooks at 1732468485634 (+48 ms)Closed at 1732468485634 2024-11-24T17:14:45,634 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732468449655.67115580ab5444880122d14f983db95c. 2024-11-24T17:14:45,634 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 89f671091abaef7d04c744da188c0bf9, disabling compactions & flushes 2024-11-24T17:14:45,634 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:45,634 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:45,634 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. after waiting 0 ms 2024-11-24T17:14:45,634 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:45,636 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:45,636 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 71e89563154e9bf392a9193da474463d: Waiting for close lock at 1732468485577Running coprocessor pre-close hooks at 1732468485577Disabling compacts and flushes for region at 1732468485577Disabling writes for close at 1732468485578 (+1 ms)Writing region close event to WAL at 1732468485591 (+13 ms)Running coprocessor post-close hooks at 1732468485636 (+45 ms)Closed at 1732468485636 2024-11-24T17:14:45,636 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732468449655.71e89563154e9bf392a9193da474463d. 2024-11-24T17:14:45,637 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6f16dc15c0e0aff7efc028181a4fdbe3, disabling compactions & flushes 2024-11-24T17:14:45,637 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:45,637 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:45,637 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. after waiting 0 ms 2024-11-24T17:14:45,637 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:45,664 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/89f671091abaef7d04c744da188c0bf9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,666 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:45,666 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 89f671091abaef7d04c744da188c0bf9: Waiting for close lock at 1732468485634Running coprocessor pre-close hooks at 1732468485634Disabling compacts and flushes for region at 1732468485634Disabling writes for close at 1732468485634Writing region close event to WAL at 1732468485639 (+5 ms)Running coprocessor post-close hooks at 1732468485666 (+27 ms)Closed at 1732468485666 2024-11-24T17:14:45,666 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732468449655.89f671091abaef7d04c744da188c0bf9. 2024-11-24T17:14:45,666 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3bf176f82dd506ad26a9f1d6ebd6f33a, disabling compactions & flushes 2024-11-24T17:14:45,666 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:45,667 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:45,667 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. after waiting 0 ms 2024-11-24T17:14:45,667 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:45,672 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/6f16dc15c0e0aff7efc028181a4fdbe3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,673 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:45,673 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6f16dc15c0e0aff7efc028181a4fdbe3: Waiting for close lock at 1732468485637Running coprocessor pre-close hooks at 1732468485637Disabling compacts and flushes for region at 1732468485637Disabling writes for close at 1732468485637Writing region close event to WAL at 1732468485648 (+11 ms)Running coprocessor post-close hooks at 1732468485673 (+25 ms)Closed at 1732468485673 2024-11-24T17:14:45,674 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732468449655.6f16dc15c0e0aff7efc028181a4fdbe3. 2024-11-24T17:14:45,674 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3963c202e600d921e58afb7a1966f4f0, disabling compactions & flushes 2024-11-24T17:14:45,674 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:45,674 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:45,674 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. after waiting 0 ms 2024-11-24T17:14:45,674 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:45,681 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1351): Waiting on 12eb6cfa7992f8b2d57701a44d5e5526, 3bf176f82dd506ad26a9f1d6ebd6f33a, 4b5321b4dd571156ebac77ca1fa8b150, 93d787842f2c5c53adaf00e6e90673e7, a61743f7cd3883457b967d55f98541e1, e88c3c1672dcd6a9395b368267add589 2024-11-24T17:14:45,685 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1351): Waiting on 3132e986d91cd07c924412b1164b0ada, 3963c202e600d921e58afb7a1966f4f0, 4a1b996a2c1476427750ae2fbe015bc5, 68cc51974c64ffdbd8bb6b53a87450da, 7ba4002d0b6663684be2c126a81b2c74, a6d2969d9f925c5b8cb0ff244090dcd4 2024-11-24T17:14:45,692 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3bf176f82dd506ad26a9f1d6ebd6f33a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,699 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:45,699 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3bf176f82dd506ad26a9f1d6ebd6f33a: Waiting for close lock at 1732468485666Running coprocessor pre-close hooks at 1732468485666Disabling compacts and flushes for region at 1732468485666Disabling writes for close at 1732468485667 (+1 ms)Writing region close event to WAL at 1732468485668 (+1 ms)Running coprocessor post-close hooks at 1732468485699 (+31 ms)Closed at 1732468485699 2024-11-24T17:14:45,699 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732468449655.3bf176f82dd506ad26a9f1d6ebd6f33a. 2024-11-24T17:14:45,699 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a61743f7cd3883457b967d55f98541e1, disabling compactions & flushes 2024-11-24T17:14:45,699 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:45,699 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:45,700 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. after waiting 0 ms 2024-11-24T17:14:45,700 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:45,723 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3963c202e600d921e58afb7a1966f4f0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,724 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:45,724 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3963c202e600d921e58afb7a1966f4f0: Waiting for close lock at 1732468485674Running coprocessor pre-close hooks at 1732468485674Disabling compacts and flushes for region at 1732468485674Disabling writes for close at 1732468485674Writing region close event to WAL at 1732468485677 (+3 ms)Running coprocessor post-close hooks at 1732468485724 (+47 ms)Closed at 1732468485724 2024-11-24T17:14:45,724 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732468449655.3963c202e600d921e58afb7a1966f4f0. 2024-11-24T17:14:45,724 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 68cc51974c64ffdbd8bb6b53a87450da, disabling compactions & flushes 2024-11-24T17:14:45,725 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:45,725 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:45,725 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. after waiting 0 ms 2024-11-24T17:14:45,725 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:45,748 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a61743f7cd3883457b967d55f98541e1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,749 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/68cc51974c64ffdbd8bb6b53a87450da/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,753 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:45,753 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 68cc51974c64ffdbd8bb6b53a87450da: Waiting for close lock at 1732468485724Running coprocessor pre-close hooks at 1732468485724Disabling compacts and flushes for region at 1732468485724Disabling writes for close at 1732468485725 (+1 ms)Writing region close event to WAL at 1732468485729 (+4 ms)Running coprocessor post-close hooks at 1732468485753 (+24 ms)Closed at 1732468485753 2024-11-24T17:14:45,753 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732468449655.68cc51974c64ffdbd8bb6b53a87450da. 2024-11-24T17:14:45,754 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a6d2969d9f925c5b8cb0ff244090dcd4, disabling compactions & flushes 2024-11-24T17:14:45,754 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:45,754 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:45,754 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. after waiting 0 ms 2024-11-24T17:14:45,754 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:45,757 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:45,757 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a61743f7cd3883457b967d55f98541e1: Waiting for close lock at 1732468485699Running coprocessor pre-close hooks at 1732468485699Disabling compacts and flushes for region at 1732468485699Disabling writes for close at 1732468485700 (+1 ms)Writing region close event to WAL at 1732468485708 (+8 ms)Running coprocessor post-close hooks at 1732468485757 (+49 ms)Closed at 1732468485757 2024-11-24T17:14:45,757 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732468449655.a61743f7cd3883457b967d55f98541e1. 2024-11-24T17:14:45,758 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e88c3c1672dcd6a9395b368267add589, disabling compactions & flushes 2024-11-24T17:14:45,758 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:45,758 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:45,758 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. after waiting 0 ms 2024-11-24T17:14:45,758 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:45,781 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/a6d2969d9f925c5b8cb0ff244090dcd4/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-24T17:14:45,781 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:45,781 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a6d2969d9f925c5b8cb0ff244090dcd4: Waiting for close lock at 1732468485754Running coprocessor pre-close hooks at 1732468485754Disabling compacts and flushes for region at 1732468485754Disabling writes for close at 1732468485754Writing region close event to WAL at 1732468485760 (+6 ms)Running coprocessor post-close hooks at 1732468485781 (+21 ms)Closed at 1732468485781 2024-11-24T17:14:45,782 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732468449655.a6d2969d9f925c5b8cb0ff244090dcd4. 2024-11-24T17:14:45,782 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4a1b996a2c1476427750ae2fbe015bc5, disabling compactions & flushes 2024-11-24T17:14:45,782 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:45,782 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:45,782 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. after waiting 0 ms 2024-11-24T17:14:45,782 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:45,789 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/e88c3c1672dcd6a9395b368267add589/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,796 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:45,796 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e88c3c1672dcd6a9395b368267add589: Waiting for close lock at 1732468485758Running coprocessor pre-close hooks at 1732468485758Disabling compacts and flushes for region at 1732468485758Disabling writes for close at 1732468485758Writing region close event to WAL at 1732468485765 (+7 ms)Running coprocessor post-close hooks at 1732468485796 (+31 ms)Closed at 1732468485796 2024-11-24T17:14:45,796 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732468449655.e88c3c1672dcd6a9395b368267add589. 2024-11-24T17:14:45,796 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12eb6cfa7992f8b2d57701a44d5e5526, disabling compactions & flushes 2024-11-24T17:14:45,796 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:45,796 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:45,796 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. after waiting 0 ms 2024-11-24T17:14:45,796 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:45,821 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/4a1b996a2c1476427750ae2fbe015bc5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,823 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:45,823 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4a1b996a2c1476427750ae2fbe015bc5: Waiting for close lock at 1732468485782Running coprocessor pre-close hooks at 1732468485782Disabling compacts and flushes for region at 1732468485782Disabling writes for close at 1732468485782Writing region close event to WAL at 1732468485789 (+7 ms)Running coprocessor post-close hooks at 1732468485823 (+34 ms)Closed at 1732468485823 2024-11-24T17:14:45,823 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732468449655.4a1b996a2c1476427750ae2fbe015bc5. 2024-11-24T17:14:45,823 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7ba4002d0b6663684be2c126a81b2c74, disabling compactions & flushes 2024-11-24T17:14:45,823 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:45,824 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:45,824 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. after waiting 0 ms 2024-11-24T17:14:45,824 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:45,824 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/12eb6cfa7992f8b2d57701a44d5e5526/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,832 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:45,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12eb6cfa7992f8b2d57701a44d5e5526: Waiting for close lock at 1732468485796Running coprocessor pre-close hooks at 1732468485796Disabling compacts and flushes for region at 1732468485796Disabling writes for close at 1732468485796Writing region close event to WAL at 1732468485804 (+8 ms)Running coprocessor post-close hooks at 1732468485832 (+28 ms)Closed at 1732468485832 2024-11-24T17:14:45,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732468449655.12eb6cfa7992f8b2d57701a44d5e5526. 2024-11-24T17:14:45,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 93d787842f2c5c53adaf00e6e90673e7, disabling compactions & flushes 2024-11-24T17:14:45,832 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:45,832 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:45,833 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. after waiting 0 ms 2024-11-24T17:14:45,833 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:45,859 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/7ba4002d0b6663684be2c126a81b2c74/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,867 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:45,867 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7ba4002d0b6663684be2c126a81b2c74: Waiting for close lock at 1732468485823Running coprocessor pre-close hooks at 1732468485823Disabling compacts and flushes for region at 1732468485823Disabling writes for close at 1732468485824 (+1 ms)Writing region close event to WAL at 1732468485828 (+4 ms)Running coprocessor post-close hooks at 1732468485867 (+39 ms)Closed at 1732468485867 2024-11-24T17:14:45,867 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732468449655.7ba4002d0b6663684be2c126a81b2c74. 2024-11-24T17:14:45,867 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3132e986d91cd07c924412b1164b0ada, disabling compactions & flushes 2024-11-24T17:14:45,867 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:45,867 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:45,867 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. after waiting 0 ms 2024-11-24T17:14:45,868 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:45,880 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/93d787842f2c5c53adaf00e6e90673e7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,881 DEBUG [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1351): Waiting on 4b5321b4dd571156ebac77ca1fa8b150, 93d787842f2c5c53adaf00e6e90673e7 2024-11-24T17:14:45,883 WARN [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150., aborting... java.io.IOException: Failed to report opened region to master: test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:14:45,883 ERROR [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server a23eb99f71f0,38279,1732468442718: Failed to open region test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ccc,1732468449655.4b5321b4dd571156ebac77ca1fa8b150. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T17:14:45,884 ERROR [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T17:14:45,884 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T17:14:45,885 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:45,885 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 93d787842f2c5c53adaf00e6e90673e7: Waiting for close lock at 1732468485832Running coprocessor pre-close hooks at 1732468485832Disabling compacts and flushes for region at 1732468485832Disabling writes for close at 1732468485833 (+1 ms)Writing region close event to WAL at 1732468485846 (+13 ms)Running coprocessor post-close hooks at 1732468485885 (+39 ms)Closed at 1732468485885 2024-11-24T17:14:45,885 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T17:14:45,885 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732468449655.93d787842f2c5c53adaf00e6e90673e7. 2024-11-24T17:14:45,886 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T17:14:45,886 DEBUG [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1351): Waiting on 3132e986d91cd07c924412b1164b0ada 2024-11-24T17:14:45,886 DEBUG [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T17:14:45,889 INFO [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 655500512 }, "NonHeapMemoryUsage": { "committed": 170852352, "init": 7667712, "max": -1, "used": 168158304 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "a23eb99f71f0", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40212, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 128, "ProcessCallTime_min": 7, "ProcessCallTime_max": 17, "ProcessCallTime_mean": 11, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 10, "ProcessCallTime_75th_percentile": 13, "ProcessCallTime_90th_percentile": 17, "ProcessCallTime_95th_percentile": 17, "ProcessCallTime_98th_percentile": 17, "ProcessCallTime_99th_percentile": 17, "ProcessCallTime_99.9th_percentile": 17, "ProcessCallTime_TimeRangeCount_3-10": 2, "ProcessCallTime_TimeRangeCount_10-30": 2, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 128, "QueueCallTime_min": 0, "QueueCallTime_max": 5, "QueueCallTime_mean": 2, "QueueCallTime_25th_percentile": 1, "QueueCallTime_median": 2, "QueueCallTime_75th_percentile": 3, "QueueCallTime_90th_percentile": 5, "QueueCallTime_95th_percentile": 5, "QueueCallTime_98th_percentile": 5, "QueueCallTime_99th_percentile": 5, "QueueCallTime_99.9th_percentile": 5, "QueueCallTime_TimeRangeCount_0-1": 2, "QueueCallTime_TimeRangeCount_1-3": 1, "QueueCallTime_TimeRangeCount_3-10": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 128, "TotalCallTime_min": 8, "TotalCallTime_max": 19, "TotalCallTime_mean": 13, "TotalCallTime_25th_percentile": 9, "TotalCallTime_median": 10, "TotalCallTime_75th_percentile": 18, "TotalCallTime_90th_percentile": 19, "TotalCallTime_95th_percentile": 19, "TotalCallTime_98th_percentile": 19, "TotalCallTime_99th_percentile": 19, "TotalCallTime_99.9th_percentile": 19, "TotalCallTime_TimeRangeCount_3-10": 2, "TotalCallTime_TimeRangeCount_10-30": 2, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 128, "ResponseSize_min": 22855, "ResponseSize_max": 22855, "ResponseSize_mean": 22855, "ResponseSize_25th_percentile": 22855, "ResponseSize_median": 22855, "ResponseSize_75th_percentile": 22855, "ResponseSize_90th_percentile": 22855, "ResponseSize_95th_percentile": 22855, "ResponseSize_98th_percentile": 22855, "ResponseSize_99th_percentile": 22855, "ResponseSize_99.9th_percentile": 22855, "ResponseSize_SizeRangeCount_10000-100000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 128, "RequestSize_min": 121, "RequestSize_max": 121, "RequestSize_mean": 121, "RequestSize_25th_percentile": 121, "RequestSize_median": 121, "RequestSize_75th_percentile": 121, "RequestSize_90th_percentile": 121, "RequestSize_95th_percentile": 121, "RequestSize_98th_percentile": 121, "RequestSize_99th_percentile": 121, "RequestSize_99.9th_percentile": 121, "RequestSize_SizeRangeCount_100-1000": 4, "sentBytes": 142997 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "a23eb99f71f0", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52152", "tag.serverName": "a23eb99f71f0,44781,1732468442596", "tag.clusterId": "866d63e0-137b-480d-bc99-aace7e6a98b3", "tag.Context": "regionserver", "tag.Hostname": "a23eb99f71f0", "regionCount": 9, "storeCount": 12, "hlogFileCount": 14, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732468442596, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3327, "localBytesRead": 3327, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 17.4, "writeRequestRatePerSecond": 4.6, "ByteBuffAllocatorHeapAllocationBytes": 38153, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T17:14:45,893 WARN [RS_OPEN_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a23eb99f71f0:40213 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a23eb99f71f0:40213 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-24T17:14:45,919 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/data/default/test/3132e986d91cd07c924412b1164b0ada/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T17:14:45,921 INFO [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:45,922 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3132e986d91cd07c924412b1164b0ada: Waiting for close lock at 1732468485867Running coprocessor pre-close hooks at 1732468485867Disabling compacts and flushes for region at 1732468485867Disabling writes for close at 1732468485867Writing region close event to WAL at 1732468485873 (+6 ms)Running coprocessor post-close hooks at 1732468485921 (+48 ms)Closed at 1732468485921 2024-11-24T17:14:45,922 DEBUG [RS_CLOSE_REGION-regionserver/a23eb99f71f0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732468449655.3132e986d91cd07c924412b1164b0ada. 2024-11-24T17:14:46,081 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(976): stopping server a23eb99f71f0,38279,1732468442718; all regions closed. 2024-11-24T17:14:46,086 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(976): stopping server a23eb99f71f0,34777,1732468442786; all regions closed. 2024-11-24T17:14:46,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741865_1041 (size=5733) 2024-11-24T17:14:46,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741865_1041 (size=5733) 2024-11-24T17:14:46,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741865_1041 (size=5733) 2024-11-24T17:14:46,095 DEBUG [RS:1;a23eb99f71f0:38279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:46,095 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T17:14:46,095 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T17:14:46,095 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.ChoreService(370): Chore service for: regionserver/a23eb99f71f0:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T17:14:46,096 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T17:14:46,096 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T17:14:46,096 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T17:14:46,096 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T17:14:46,096 INFO [RS:1;a23eb99f71f0:38279 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38279 2024-11-24T17:14:46,097 INFO [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T17:14:46,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741868_1044 (size=4204) 2024-11-24T17:14:46,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741868_1044 (size=4204) 2024-11-24T17:14:46,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741868_1044 (size=4204) 2024-11-24T17:14:46,133 DEBUG [RS:2;a23eb99f71f0:34777 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs 2024-11-24T17:14:46,133 INFO [RS:2;a23eb99f71f0:34777 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a23eb99f71f0%2C34777%2C1732468442786.meta:.meta(num 1732468480395) 2024-11-24T17:14:46,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741833_1009 (size=7064) 2024-11-24T17:14:46,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741833_1009 (size=7064) 2024-11-24T17:14:46,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741833_1009 (size=7064) 2024-11-24T17:14:46,163 DEBUG [RS:2;a23eb99f71f0:34777 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/oldWALs 2024-11-24T17:14:46,163 INFO [RS:2;a23eb99f71f0:34777 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a23eb99f71f0%2C34777%2C1732468442786:(num 1732468446703) 2024-11-24T17:14:46,163 DEBUG [RS:2;a23eb99f71f0:34777 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T17:14:46,163 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T17:14:46,163 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T17:14:46,164 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.ChoreService(370): Chore service for: regionserver/a23eb99f71f0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T17:14:46,164 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T17:14:46,164 INFO [RS:2;a23eb99f71f0:34777 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34777 2024-11-24T17:14:46,164 INFO [regionserver/a23eb99f71f0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T17:14:46,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T17:14:46,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a23eb99f71f0,38279,1732468442718 2024-11-24T17:14:46,180 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T17:14:46,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a23eb99f71f0,34777,1732468442786 2024-11-24T17:14:46,222 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T17:14:46,222 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$362/0x00007f5d48900290@3ec17061 rejected from java.util.concurrent.ThreadPoolExecutor@7bc3e9dd[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 39] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-24T17:14:46,301 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a23eb99f71f0,34777,1732468442786] 2024-11-24T17:14:46,386 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a23eb99f71f0,34777,1732468442786 already deleted, retry=false 2024-11-24T17:14:46,386 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a23eb99f71f0,34777,1732468442786 expired; onlineServers=1 2024-11-24T17:14:46,386 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a23eb99f71f0,38279,1732468442718] 2024-11-24T17:14:46,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:14:46,402 INFO [RS:1;a23eb99f71f0:38279 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T17:14:46,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38279-0x1016dd6789d0002, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:14:46,402 INFO [RS:1;a23eb99f71f0:38279 {}] regionserver.HRegionServer(1031): Exiting; stopping=a23eb99f71f0,38279,1732468442718; zookeeper connection closed. 2024-11-24T17:14:46,402 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@9722a2e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@9722a2e 2024-11-24T17:14:46,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:14:46,422 INFO [RS:2;a23eb99f71f0:34777 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T17:14:46,422 INFO [RS:2;a23eb99f71f0:34777 {}] regionserver.HRegionServer(1031): Exiting; stopping=a23eb99f71f0,34777,1732468442786; zookeeper connection closed. 2024-11-24T17:14:46,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34777-0x1016dd6789d0003, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T17:14:46,423 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@c6db058 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@c6db058 2024-11-24T17:14:46,423 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T17:14:46,475 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a23eb99f71f0,38279,1732468442718 already deleted, retry=false 2024-11-24T17:14:46,476 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a23eb99f71f0,38279,1732468442718 expired; onlineServers=0 2024-11-24T17:14:46,476 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a23eb99f71f0,40213,1732468441272' ***** 2024-11-24T17:14:46,476 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T17:14:46,476 INFO [M:0;a23eb99f71f0:40213 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T17:14:46,476 INFO [M:0;a23eb99f71f0:40213 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T17:14:46,476 DEBUG [M:0;a23eb99f71f0:40213 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T17:14:46,476 DEBUG [M:0;a23eb99f71f0:40213 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T17:14:46,476 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T17:14:46,476 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.small.0-1732468445517 {}] cleaner.HFileCleaner(306): Exit Thread[master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.small.0-1732468445517,5,FailOnTimeoutGroup] 2024-11-24T17:14:46,476 DEBUG [master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.large.0-1732468445512 {}] cleaner.HFileCleaner(306): Exit Thread[master/a23eb99f71f0:0:becomeActiveMaster-HFileCleaner.large.0-1732468445512,5,FailOnTimeoutGroup] 2024-11-24T17:14:46,477 INFO [M:0;a23eb99f71f0:40213 {}] hbase.ChoreService(370): Chore service for: master/a23eb99f71f0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T17:14:46,477 INFO [M:0;a23eb99f71f0:40213 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T17:14:46,477 DEBUG [M:0;a23eb99f71f0:40213 {}] master.HMaster(1795): Stopping service threads 2024-11-24T17:14:46,477 INFO [M:0;a23eb99f71f0:40213 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T17:14:46,477 INFO [M:0;a23eb99f71f0:40213 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T17:14:46,477 INFO [M:0;a23eb99f71f0:40213 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T17:14:46,477 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T17:14:46,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T17:14:46,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T17:14:46,583 DEBUG [M:0;a23eb99f71f0:40213 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-24T17:14:46,584 DEBUG [M:0;a23eb99f71f0:40213 {}] master.ActiveMasterManager(353): master:40213-0x1016dd6789d0000, quorum=127.0.0.1:52152, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-24T17:14:46,585 INFO [M:0;a23eb99f71f0:40213 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/.lastflushedseqids 2024-11-24T17:14:46,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741873_1049 (size=1764) 2024-11-24T17:14:46,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741873_1049 (size=1764) 2024-11-24T17:14:46,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741873_1049 (size=1764) 2024-11-24T17:14:46,610 INFO [M:0;a23eb99f71f0:40213 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T17:14:46,611 INFO [M:0;a23eb99f71f0:40213 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T17:14:46,612 DEBUG [M:0;a23eb99f71f0:40213 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T17:14:46,612 INFO [M:0;a23eb99f71f0:40213 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:14:46,612 DEBUG [M:0;a23eb99f71f0:40213 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:14:46,612 DEBUG [M:0;a23eb99f71f0:40213 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T17:14:46,612 DEBUG [M:0;a23eb99f71f0:40213 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T17:14:46,612 INFO [M:0;a23eb99f71f0:40213 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=258.59 KB heapSize=314.41 KB 2024-11-24T17:14:46,639 DEBUG [M:0;a23eb99f71f0:40213 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/90451611920d49d2a1719e041f50a190 is 82, key is hbase:meta,,1/info:regioninfo/1732468480767/Put/seqid=0 2024-11-24T17:14:46,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37919 is added to blk_1073741874_1050 (size=6071) 2024-11-24T17:14:46,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45925 is added to blk_1073741874_1050 (size=6071) 2024-11-24T17:14:46,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37313 is added to blk_1073741874_1050 (size=6071) 2024-11-24T17:14:46,653 INFO [M:0;a23eb99f71f0:40213 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/90451611920d49d2a1719e041f50a190 2024-11-24T17:14:46,683 DEBUG [M:0;a23eb99f71f0:40213 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44865/user/jenkins/test-data/dff0f0cc-10f9-8eb1-1b57-55e474bf44f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4575e3da011540eb85362f22c18f4173 is 2990, key is \x00\x00\x00\x00\x00\x00\x00