2024-11-20 11:14:17,139 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b 2024-11-20 11:14:17,156 main DEBUG Took 0.014734 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 11:14:17,157 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 11:14:17,157 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 11:14:17,159 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 11:14:17,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,190 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 11:14:17,209 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,211 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,213 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,213 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,214 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,217 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,218 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,219 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,220 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,221 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,222 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,222 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,223 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,224 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,225 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 11:14:17,226 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,226 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 11:14:17,228 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 11:14:17,230 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 11:14:17,232 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 11:14:17,233 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 11:14:17,234 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 11:14:17,234 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 11:14:17,246 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 11:14:17,249 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 11:14:17,251 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 11:14:17,251 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 11:14:17,252 main DEBUG createAppenders(={Console}) 2024-11-20 11:14:17,253 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b initialized 2024-11-20 11:14:17,253 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b 2024-11-20 11:14:17,254 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b OK. 2024-11-20 11:14:17,254 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 11:14:17,255 main DEBUG OutputStream closed 2024-11-20 11:14:17,255 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 11:14:17,255 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 11:14:17,256 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK 2024-11-20 11:14:17,367 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 11:14:17,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 11:14:17,371 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 11:14:17,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 11:14:17,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 11:14:17,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 11:14:17,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 11:14:17,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 11:14:17,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 11:14:17,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 11:14:17,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 11:14:17,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 11:14:17,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 11:14:17,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 11:14:17,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 11:14:17,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 11:14:17,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 11:14:17,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 11:14:17,383 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 11:14:17,383 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@539d019) with optional ClassLoader: null 2024-11-20 11:14:17,384 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 11:14:17,385 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@539d019] started OK. 2024-11-20T11:14:17,402 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-20 11:14:17,405 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 11:14:17,405 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T11:14:17,791 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7 2024-11-20T11:14:17,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T11:14:17,840 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5, deleteOnExit=true 2024-11-20T11:14:17,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T11:14:17,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/test.cache.data in system properties and HBase conf 2024-11-20T11:14:17,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T11:14:17,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir in system properties and HBase conf 2024-11-20T11:14:17,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T11:14:17,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T11:14:17,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T11:14:17,975 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T11:14:18,180 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T11:14:18,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T11:14:18,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T11:14:18,196 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T11:14:18,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T11:14:18,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T11:14:18,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T11:14:18,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T11:14:18,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T11:14:18,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T11:14:18,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/nfs.dump.dir in system properties and HBase conf 2024-11-20T11:14:18,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/java.io.tmpdir in system properties and HBase conf 2024-11-20T11:14:18,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T11:14:18,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T11:14:18,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T11:14:19,453 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T11:14:19,553 INFO [Time-limited test {}] log.Log(170): Logging initialized @3556ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T11:14:19,661 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T11:14:19,740 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T11:14:19,806 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T11:14:19,806 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T11:14:19,808 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T11:14:19,822 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T11:14:19,827 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@952689a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir/,AVAILABLE} 2024-11-20T11:14:19,831 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@378c34a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T11:14:20,097 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@302cfe51{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/java.io.tmpdir/jetty-localhost-33855-hadoop-hdfs-3_4_1-tests_jar-_-any-15229658452539231823/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T11:14:20,113 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c1eb5a9{HTTP/1.1, (http/1.1)}{localhost:33855} 2024-11-20T11:14:20,113 INFO [Time-limited test {}] server.Server(415): Started @4118ms 2024-11-20T11:14:20,619 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T11:14:20,635 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T11:14:20,637 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T11:14:20,640 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T11:14:20,640 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T11:14:20,641 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18a3047c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir/,AVAILABLE} 2024-11-20T11:14:20,642 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62c0eab0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T11:14:20,807 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a0ce16d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/java.io.tmpdir/jetty-localhost-37095-hadoop-hdfs-3_4_1-tests_jar-_-any-10952467162575476768/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T11:14:20,808 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1cafe9ff{HTTP/1.1, (http/1.1)}{localhost:37095} 2024-11-20T11:14:20,809 INFO [Time-limited test {}] server.Server(415): Started @4813ms 2024-11-20T11:14:20,886 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T11:14:21,063 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T11:14:21,069 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T11:14:21,080 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T11:14:21,080 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T11:14:21,081 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T11:14:21,082 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@414e5e95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir/,AVAILABLE} 2024-11-20T11:14:21,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a49f6d7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T11:14:21,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4caee23f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/java.io.tmpdir/jetty-localhost-46561-hadoop-hdfs-3_4_1-tests_jar-_-any-3757214912570777061/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T11:14:21,253 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@178c0f89{HTTP/1.1, (http/1.1)}{localhost:46561} 2024-11-20T11:14:21,253 INFO [Time-limited test {}] server.Server(415): Started @5258ms 2024-11-20T11:14:21,256 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T11:14:21,333 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T11:14:21,341 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T11:14:21,365 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T11:14:21,365 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T11:14:21,366 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T11:14:21,368 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@60467e4f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir/,AVAILABLE} 2024-11-20T11:14:21,369 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e0151e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T11:14:21,527 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data1/current/BP-1883451983-172.17.0.2-1732101259138/current, will proceed with Du for space computation calculation, 2024-11-20T11:14:21,536 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data2/current/BP-1883451983-172.17.0.2-1732101259138/current, will proceed with Du for space computation calculation, 2024-11-20T11:14:21,539 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data3/current/BP-1883451983-172.17.0.2-1732101259138/current, will proceed with Du for space computation calculation, 2024-11-20T11:14:21,547 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data4/current/BP-1883451983-172.17.0.2-1732101259138/current, will proceed with Du for space computation calculation, 2024-11-20T11:14:21,551 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@459058f9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/java.io.tmpdir/jetty-localhost-35437-hadoop-hdfs-3_4_1-tests_jar-_-any-5274520785893319280/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T11:14:21,557 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@38c64c15{HTTP/1.1, (http/1.1)}{localhost:35437} 2024-11-20T11:14:21,557 INFO [Time-limited test {}] server.Server(415): Started @5561ms 2024-11-20T11:14:21,559 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T11:14:21,642 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T11:14:21,644 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T11:14:21,746 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xde85489a75e0fb43 with lease ID 0xad00996df43cb50c: Processing first storage report for DS-dfc6e092-f541-494b-81e4-826b95dc8a56 from datanode DatanodeRegistration(127.0.0.1:37351, datanodeUuid=3366fa75-d333-43d9-a106-ab42a7015d7e, infoPort=40607, infoSecurePort=0, ipcPort=44099, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138) 2024-11-20T11:14:21,747 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xde85489a75e0fb43 with lease ID 0xad00996df43cb50c: from storage DS-dfc6e092-f541-494b-81e4-826b95dc8a56 node DatanodeRegistration(127.0.0.1:37351, datanodeUuid=3366fa75-d333-43d9-a106-ab42a7015d7e, infoPort=40607, infoSecurePort=0, ipcPort=44099, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T11:14:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed33116d9a63aeaf with lease ID 0xad00996df43cb50b: Processing first storage report for DS-ef273f42-ddeb-44b0-90ec-87c34e9d8141 from datanode DatanodeRegistration(127.0.0.1:42077, datanodeUuid=d526301a-a1e1-4635-a36e-ab3af0167801, infoPort=40685, infoSecurePort=0, ipcPort=45229, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138) 2024-11-20T11:14:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed33116d9a63aeaf with lease ID 0xad00996df43cb50b: from storage DS-ef273f42-ddeb-44b0-90ec-87c34e9d8141 node DatanodeRegistration(127.0.0.1:42077, datanodeUuid=d526301a-a1e1-4635-a36e-ab3af0167801, infoPort=40685, infoSecurePort=0, ipcPort=45229, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T11:14:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed33116d9a63aeaf with lease ID 0xad00996df43cb50b: Processing first storage report for DS-229a9e4e-d177-4db5-89cf-82d905ac2df8 from datanode DatanodeRegistration(127.0.0.1:42077, datanodeUuid=d526301a-a1e1-4635-a36e-ab3af0167801, infoPort=40685, infoSecurePort=0, ipcPort=45229, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138) 2024-11-20T11:14:21,749 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed33116d9a63aeaf with lease ID 0xad00996df43cb50b: from storage DS-229a9e4e-d177-4db5-89cf-82d905ac2df8 node DatanodeRegistration(127.0.0.1:42077, datanodeUuid=d526301a-a1e1-4635-a36e-ab3af0167801, infoPort=40685, infoSecurePort=0, ipcPort=45229, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T11:14:21,749 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xde85489a75e0fb43 with lease ID 0xad00996df43cb50c: Processing first storage report for DS-8a9c1aea-8630-4011-842c-e93d17cde1f3 from datanode DatanodeRegistration(127.0.0.1:37351, datanodeUuid=3366fa75-d333-43d9-a106-ab42a7015d7e, infoPort=40607, infoSecurePort=0, ipcPort=44099, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138) 2024-11-20T11:14:21,749 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xde85489a75e0fb43 with lease ID 0xad00996df43cb50c: from storage DS-8a9c1aea-8630-4011-842c-e93d17cde1f3 node DatanodeRegistration(127.0.0.1:37351, datanodeUuid=3366fa75-d333-43d9-a106-ab42a7015d7e, infoPort=40607, infoSecurePort=0, ipcPort=44099, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T11:14:21,818 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data5/current/BP-1883451983-172.17.0.2-1732101259138/current, will proceed with Du for space computation calculation, 2024-11-20T11:14:21,821 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data6/current/BP-1883451983-172.17.0.2-1732101259138/current, will proceed with Du for space computation calculation, 2024-11-20T11:14:21,918 WARN [Thread-123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T11:14:21,948 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb836d1a0b90dbffe with lease ID 0xad00996df43cb50d: Processing first storage report for DS-5f4458a9-3703-4eaa-8233-a851e4aaf890 from datanode DatanodeRegistration(127.0.0.1:44787, datanodeUuid=c1f15936-2099-4e63-8ecb-172c484b2c71, infoPort=44047, infoSecurePort=0, ipcPort=33515, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138) 2024-11-20T11:14:21,948 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb836d1a0b90dbffe with lease ID 0xad00996df43cb50d: from storage DS-5f4458a9-3703-4eaa-8233-a851e4aaf890 node DatanodeRegistration(127.0.0.1:44787, datanodeUuid=c1f15936-2099-4e63-8ecb-172c484b2c71, infoPort=44047, infoSecurePort=0, ipcPort=33515, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T11:14:21,949 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb836d1a0b90dbffe with lease ID 0xad00996df43cb50d: Processing first storage report for DS-be2e280b-bde4-4d87-8958-c4106f52fbc4 from datanode DatanodeRegistration(127.0.0.1:44787, datanodeUuid=c1f15936-2099-4e63-8ecb-172c484b2c71, infoPort=44047, infoSecurePort=0, ipcPort=33515, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138) 2024-11-20T11:14:21,949 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb836d1a0b90dbffe with lease ID 0xad00996df43cb50d: from storage DS-be2e280b-bde4-4d87-8958-c4106f52fbc4 node DatanodeRegistration(127.0.0.1:44787, datanodeUuid=c1f15936-2099-4e63-8ecb-172c484b2c71, infoPort=44047, infoSecurePort=0, ipcPort=33515, storageInfo=lv=-57;cid=testClusterID;nsid=1008437996;c=1732101259138), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T11:14:22,202 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7 2024-11-20T11:14:22,322 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/zookeeper_0, clientPort=55317, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T11:14:22,335 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55317 2024-11-20T11:14:22,360 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:22,370 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:22,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741825_1001 (size=7) 2024-11-20T11:14:22,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741825_1001 (size=7) 2024-11-20T11:14:22,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741825_1001 (size=7) 2024-11-20T11:14:23,162 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a with version=8 2024-11-20T11:14:23,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/hbase-staging 2024-11-20T11:14:23,294 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T11:14:23,635 INFO [Time-limited test {}] client.ConnectionUtils(128): master/448ea6c54a8c:0 server-side Connection retries=45 2024-11-20T11:14:23,652 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:23,653 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:23,659 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T11:14:23,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:23,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T11:14:23,857 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T11:14:23,936 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T11:14:23,946 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T11:14:23,952 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T11:14:23,985 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7694 (auto-detected) 2024-11-20T11:14:23,986 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T11:14:24,014 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38459 2024-11-20T11:14:24,041 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38459 connecting to ZooKeeper ensemble=127.0.0.1:55317 2024-11-20T11:14:24,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384590x0, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T11:14:24,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38459-0x1013515b6630000 connected 2024-11-20T11:14:24,141 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,144 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,156 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T11:14:24,161 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a, hbase.cluster.distributed=false 2024-11-20T11:14:24,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T11:14:24,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38459 2024-11-20T11:14:24,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38459 2024-11-20T11:14:24,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38459 2024-11-20T11:14:24,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38459 2024-11-20T11:14:24,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38459 2024-11-20T11:14:24,341 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/448ea6c54a8c:0 server-side Connection retries=45 2024-11-20T11:14:24,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,343 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T11:14:24,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T11:14:24,347 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T11:14:24,351 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T11:14:24,352 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43507 2024-11-20T11:14:24,355 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43507 connecting to ZooKeeper ensemble=127.0.0.1:55317 2024-11-20T11:14:24,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,362 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:435070x0, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T11:14:24,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:435070x0, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T11:14:24,379 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T11:14:24,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43507-0x1013515b6630001 connected 2024-11-20T11:14:24,389 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T11:14:24,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T11:14:24,399 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T11:14:24,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43507 2024-11-20T11:14:24,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43507 2024-11-20T11:14:24,410 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43507 2024-11-20T11:14:24,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43507 2024-11-20T11:14:24,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43507 2024-11-20T11:14:24,438 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/448ea6c54a8c:0 server-side Connection retries=45 2024-11-20T11:14:24,438 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,438 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,439 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T11:14:24,439 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,439 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T11:14:24,439 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T11:14:24,440 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T11:14:24,441 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35099 2024-11-20T11:14:24,443 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35099 connecting to ZooKeeper ensemble=127.0.0.1:55317 2024-11-20T11:14:24,444 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,449 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350990x0, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T11:14:24,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:350990x0, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T11:14:24,461 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35099-0x1013515b6630002 connected 2024-11-20T11:14:24,462 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T11:14:24,468 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T11:14:24,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T11:14:24,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T11:14:24,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35099 2024-11-20T11:14:24,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35099 2024-11-20T11:14:24,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35099 2024-11-20T11:14:24,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35099 2024-11-20T11:14:24,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35099 2024-11-20T11:14:24,505 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/448ea6c54a8c:0 server-side Connection retries=45 2024-11-20T11:14:24,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,506 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T11:14:24,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T11:14:24,507 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T11:14:24,507 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T11:14:24,507 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T11:14:24,508 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35491 2024-11-20T11:14:24,511 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35491 connecting to ZooKeeper ensemble=127.0.0.1:55317 2024-11-20T11:14:24,513 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,516 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354910x0, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T11:14:24,526 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:354910x0, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T11:14:24,527 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T11:14:24,533 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35491-0x1013515b6630003 connected 2024-11-20T11:14:24,537 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T11:14:24,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T11:14:24,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T11:14:24,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35491 2024-11-20T11:14:24,548 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35491 2024-11-20T11:14:24,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35491 2024-11-20T11:14:24,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35491 2024-11-20T11:14:24,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35491 2024-11-20T11:14:24,571 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;448ea6c54a8c:38459 2024-11-20T11:14:24,573 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:24,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T11:14:24,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T11:14:24,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T11:14:24,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T11:14:24,588 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:24,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T11:14:24,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T11:14:24,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:24,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T11:14:24,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:24,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:24,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:24,624 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T11:14:24,626 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/448ea6c54a8c,38459,1732101263372 from backup master directory 2024-11-20T11:14:24,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:24,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T11:14:24,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T11:14:24,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T11:14:24,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T11:14:24,632 WARN [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T11:14:24,632 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:24,635 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T11:14:24,637 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T11:14:24,702 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/hbase.id] with ID: 486e3fde-64a7-493d-99c8-877072beb5df 2024-11-20T11:14:24,703 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/.tmp/hbase.id 2024-11-20T11:14:24,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741826_1002 (size=42) 2024-11-20T11:14:24,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741826_1002 (size=42) 2024-11-20T11:14:24,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741826_1002 (size=42) 2024-11-20T11:14:24,741 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/.tmp/hbase.id]:[hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/hbase.id] 2024-11-20T11:14:24,802 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T11:14:24,808 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T11:14:24,838 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-20T11:14:24,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:24,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:24,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:24,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:24,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741827_1003 (size=196) 2024-11-20T11:14:24,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741827_1003 (size=196) 2024-11-20T11:14:24,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741827_1003 (size=196) 2024-11-20T11:14:24,901 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T11:14:24,903 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T11:14:24,924 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T11:14:24,930 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T11:14:24,963 WARN [IPC Server handler 2 on default port 35273 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T11:14:24,964 WARN [IPC Server handler 2 on default port 35273 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T11:14:24,964 WARN [IPC Server handler 2 on default port 35273 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T11:14:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741828_1004 (size=1189) 2024-11-20T11:14:24,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741828_1004 (size=1189) 2024-11-20T11:14:25,401 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store 2024-11-20T11:14:25,418 WARN [IPC Server handler 4 on default port 35273 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T11:14:25,418 WARN [IPC Server handler 4 on default port 35273 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T11:14:25,419 WARN [IPC Server handler 4 on default port 35273 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T11:14:25,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741829_1005 (size=34) 2024-11-20T11:14:25,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741829_1005 (size=34) 2024-11-20T11:14:25,435 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T11:14:25,438 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:25,440 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T11:14:25,440 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T11:14:25,440 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T11:14:25,442 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T11:14:25,442 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T11:14:25,442 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T11:14:25,443 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732101265439Disabling compacts and flushes for region at 1732101265439Disabling writes for close at 1732101265442 (+3 ms)Writing region close event to WAL at 1732101265442Closed at 1732101265442 2024-11-20T11:14:25,447 WARN [master/448ea6c54a8c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/.initializing 2024-11-20T11:14:25,447 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/WALs/448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:25,460 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T11:14:25,482 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=448ea6c54a8c%2C38459%2C1732101263372, suffix=, logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/WALs/448ea6c54a8c,38459,1732101263372, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/oldWALs, maxLogs=10 2024-11-20T11:14:25,517 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/WALs/448ea6c54a8c,38459,1732101263372/448ea6c54a8c%2C38459%2C1732101263372.1732101265488, exclude list is [], retry=0 2024-11-20T11:14:25,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44787,DS-5f4458a9-3703-4eaa-8233-a851e4aaf890,DISK] 2024-11-20T11:14:25,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42077,DS-ef273f42-ddeb-44b0-90ec-87c34e9d8141,DISK] 2024-11-20T11:14:25,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37351,DS-dfc6e092-f541-494b-81e4-826b95dc8a56,DISK] 2024-11-20T11:14:25,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T11:14:25,598 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/WALs/448ea6c54a8c,38459,1732101263372/448ea6c54a8c%2C38459%2C1732101263372.1732101265488 2024-11-20T11:14:25,599 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44047:44047),(127.0.0.1/127.0.0.1:40685:40685),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T11:14:25,599 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T11:14:25,600 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:25,604 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,605 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T11:14:25,690 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:25,694 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:25,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T11:14:25,699 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:25,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:25,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,705 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T11:14:25,705 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:25,707 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:25,707 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T11:14:25,713 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:25,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:25,715 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,720 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,722 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,729 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,729 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,736 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T11:14:25,741 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T11:14:25,747 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:25,750 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73715259, jitterRate=0.09844295680522919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T11:14:25,759 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732101265624Initializing all the Stores at 1732101265627 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101265627Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101265628 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101265629 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101265629Cleaning up temporary data from old regions at 1732101265730 (+101 ms)Region opened successfully at 1732101265759 (+29 ms) 2024-11-20T11:14:25,761 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T11:14:25,810 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c82091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=448ea6c54a8c/172.17.0.2:0 2024-11-20T11:14:25,858 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T11:14:25,875 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:25,875 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:25,879 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:25,881 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T11:14:25,888 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T11:14:25,888 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T11:14:25,925 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T11:14:25,935 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T11:14:25,937 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T11:14:25,940 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T11:14:25,942 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T11:14:25,944 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T11:14:25,947 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T11:14:25,952 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T11:14:25,954 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T11:14:25,956 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T11:14:25,958 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T11:14:25,979 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T11:14:25,981 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T11:14:25,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T11:14:25,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T11:14:25,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:25,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:25,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T11:14:25,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:25,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T11:14:25,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:25,991 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=448ea6c54a8c,38459,1732101263372, sessionid=0x1013515b6630000, setting cluster-up flag (Was=false) 2024-11-20T11:14:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:26,013 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T11:14:26,015 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:26,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:26,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:26,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:26,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:26,033 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T11:14:26,035 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:26,044 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T11:14:26,059 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(746): ClusterId : 486e3fde-64a7-493d-99c8-877072beb5df 2024-11-20T11:14:26,059 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(746): ClusterId : 486e3fde-64a7-493d-99c8-877072beb5df 2024-11-20T11:14:26,059 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(746): ClusterId : 486e3fde-64a7-493d-99c8-877072beb5df 2024-11-20T11:14:26,063 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T11:14:26,063 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T11:14:26,063 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T11:14:26,070 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T11:14:26,070 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T11:14:26,070 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T11:14:26,070 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T11:14:26,070 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T11:14:26,070 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T11:14:26,078 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T11:14:26,078 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T11:14:26,078 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T11:14:26,079 DEBUG [RS:0;448ea6c54a8c:43507 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e96b49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=448ea6c54a8c/172.17.0.2:0 2024-11-20T11:14:26,079 DEBUG [RS:2;448ea6c54a8c:35491 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de3a06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=448ea6c54a8c/172.17.0.2:0 2024-11-20T11:14:26,079 DEBUG [RS:1;448ea6c54a8c:35099 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55407a58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=448ea6c54a8c/172.17.0.2:0 2024-11-20T11:14:26,099 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;448ea6c54a8c:35099 2024-11-20T11:14:26,113 DEBUG [RS:2;448ea6c54a8c:35491 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;448ea6c54a8c:35491 2024-11-20T11:14:26,113 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T11:14:26,114 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T11:14:26,114 DEBUG [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T11:14:26,115 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T11:14:26,115 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T11:14:26,115 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T11:14:26,118 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(2659): reportForDuty to master=448ea6c54a8c,38459,1732101263372 with port=35491, startcode=1732101264505 2024-11-20T11:14:26,119 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(2659): reportForDuty to master=448ea6c54a8c,38459,1732101263372 with port=35099, startcode=1732101264437 2024-11-20T11:14:26,122 DEBUG [RS:0;448ea6c54a8c:43507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;448ea6c54a8c:43507 2024-11-20T11:14:26,122 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T11:14:26,122 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T11:14:26,122 DEBUG [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T11:14:26,123 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(2659): reportForDuty to master=448ea6c54a8c,38459,1732101263372 with port=43507, startcode=1732101264298 2024-11-20T11:14:26,133 DEBUG [RS:2;448ea6c54a8c:35491 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T11:14:26,134 DEBUG [RS:0;448ea6c54a8c:43507 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T11:14:26,137 DEBUG [RS:1;448ea6c54a8c:35099 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T11:14:26,194 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:26,200 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42751, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T11:14:26,201 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46291, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T11:14:26,201 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55075, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T11:14:26,209 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:26,212 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T11:14:26,219 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:26,223 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:26,224 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T11:14:26,235 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 448ea6c54a8c,38459,1732101263372 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T11:14:26,244 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/448ea6c54a8c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T11:14:26,245 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/448ea6c54a8c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T11:14:26,245 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/448ea6c54a8c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T11:14:26,245 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/448ea6c54a8c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T11:14:26,245 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/448ea6c54a8c:0, corePoolSize=10, maxPoolSize=10 2024-11-20T11:14:26,246 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T11:14:26,246 DEBUG [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T11:14:26,246 DEBUG [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T11:14:26,246 WARN [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T11:14:26,246 WARN [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T11:14:26,246 WARN [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T11:14:26,246 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,246 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/448ea6c54a8c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T11:14:26,246 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,255 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101296255 2024-11-20T11:14:26,262 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:26,262 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T11:14:26,268 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T11:14:26,269 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T11:14:26,274 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T11:14:26,274 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T11:14:26,275 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T11:14:26,275 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T11:14:26,277 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:26,277 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,277 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T11:14:26,293 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T11:14:26,294 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T11:14:26,295 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T11:14:26,307 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T11:14:26,307 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T11:14:26,316 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/448ea6c54a8c:0:becomeActiveMaster-HFileCleaner.large.0-1732101266310,5,FailOnTimeoutGroup] 2024-11-20T11:14:26,322 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/448ea6c54a8c:0:becomeActiveMaster-HFileCleaner.small.0-1732101266317,5,FailOnTimeoutGroup] 2024-11-20T11:14:26,322 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,323 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T11:14:26,324 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,325 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,347 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(2659): reportForDuty to master=448ea6c54a8c,38459,1732101263372 with port=35099, startcode=1732101264437 2024-11-20T11:14:26,348 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(2659): reportForDuty to master=448ea6c54a8c,38459,1732101263372 with port=43507, startcode=1732101264298 2024-11-20T11:14:26,348 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(2659): reportForDuty to master=448ea6c54a8c,38459,1732101263372 with port=35491, startcode=1732101264505 2024-11-20T11:14:26,353 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:26,356 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] master.ServerManager(517): Registering regionserver=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:26,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741831_1007 (size=1321) 2024-11-20T11:14:26,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741831_1007 (size=1321) 2024-11-20T11:14:26,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741831_1007 (size=1321) 2024-11-20T11:14:26,369 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:26,369 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35273 2024-11-20T11:14:26,369 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:26,369 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T11:14:26,369 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] master.ServerManager(517): Registering regionserver=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:26,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T11:14:26,376 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:26,376 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] master.ServerManager(517): Registering regionserver=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:26,376 DEBUG [RS:1;448ea6c54a8c:35099 {}] zookeeper.ZKUtil(111): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:26,376 WARN [RS:1;448ea6c54a8c:35099 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T11:14:26,376 DEBUG [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:26,376 DEBUG [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35273 2024-11-20T11:14:26,377 DEBUG [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T11:14:26,377 INFO [RS:1;448ea6c54a8c:35099 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T11:14:26,380 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:26,386 DEBUG [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:26,386 DEBUG [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35273 2024-11-20T11:14:26,386 DEBUG [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T11:14:26,387 DEBUG [RS:2;448ea6c54a8c:35491 {}] zookeeper.ZKUtil(111): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:26,387 WARN [RS:2;448ea6c54a8c:35491 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T11:14:26,387 INFO [RS:2;448ea6c54a8c:35491 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T11:14:26,387 DEBUG [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:26,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T11:14:26,390 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [448ea6c54a8c,35099,1732101264437] 2024-11-20T11:14:26,390 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [448ea6c54a8c,35491,1732101264505] 2024-11-20T11:14:26,390 DEBUG [RS:0;448ea6c54a8c:43507 {}] zookeeper.ZKUtil(111): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:26,391 WARN [RS:0;448ea6c54a8c:43507 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T11:14:26,391 INFO [RS:0;448ea6c54a8c:43507 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T11:14:26,391 DEBUG [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:26,391 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [448ea6c54a8c,43507,1732101264298] 2024-11-20T11:14:26,425 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T11:14:26,425 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T11:14:26,426 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T11:14:26,446 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T11:14:26,447 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T11:14:26,450 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T11:14:26,457 INFO [RS:1;448ea6c54a8c:35099 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T11:14:26,457 INFO [RS:2;448ea6c54a8c:35491 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T11:14:26,457 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,460 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,460 INFO [RS:0;448ea6c54a8c:43507 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T11:14:26,460 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,461 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T11:14:26,464 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T11:14:26,474 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T11:14:26,474 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T11:14:26,475 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T11:14:26,475 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T11:14:26,475 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T11:14:26,475 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T11:14:26,476 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,476 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T11:14:26,477 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T11:14:26,477 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,477 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T11:14:26,477 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,477 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T11:14:26,477 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,477 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T11:14:26,478 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,478 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,478 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T11:14:26,478 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,478 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,478 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T11:14:26,479 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,479 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,479 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,479 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,479 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,479 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,479 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,479 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,480 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/448ea6c54a8c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T11:14:26,480 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,480 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,480 DEBUG [RS:2;448ea6c54a8c:35491 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/448ea6c54a8c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T11:14:26,480 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T11:14:26,480 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,480 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,480 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,481 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,481 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,481 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,481 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/448ea6c54a8c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T11:14:26,481 DEBUG [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/448ea6c54a8c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T11:14:26,476 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,483 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,483 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,483 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,484 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,484 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,484 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T11:14:26,484 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,484 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,484 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,487 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,487 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,487 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/448ea6c54a8c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T11:14:26,487 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/448ea6c54a8c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T11:14:26,487 DEBUG [RS:0;448ea6c54a8c:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/448ea6c54a8c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T11:14:26,512 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,513 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,513 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,513 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,513 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,513 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,513 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,513 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,513 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,35099,1732101264437-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T11:14:26,514 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,514 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,514 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,35491,1732101264505-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T11:14:26,518 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,519 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,519 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,519 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,519 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,519 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,43507,1732101264298-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T11:14:26,549 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T11:14:26,549 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T11:14:26,550 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T11:14:26,551 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,35099,1732101264437-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,551 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,43507,1732101264298-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,552 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,552 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.Replication(171): 448ea6c54a8c,35099,1732101264437 started 2024-11-20T11:14:26,552 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,552 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.Replication(171): 448ea6c54a8c,43507,1732101264298 started 2024-11-20T11:14:26,553 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,35491,1732101264505-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,553 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,554 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.Replication(171): 448ea6c54a8c,35491,1732101264505 started 2024-11-20T11:14:26,576 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,577 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1482): Serving as 448ea6c54a8c,35099,1732101264437, RpcServer on 448ea6c54a8c/172.17.0.2:35099, sessionid=0x1013515b6630002 2024-11-20T11:14:26,577 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,577 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(1482): Serving as 448ea6c54a8c,35491,1732101264505, RpcServer on 448ea6c54a8c/172.17.0.2:35491, sessionid=0x1013515b6630003 2024-11-20T11:14:26,578 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T11:14:26,578 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T11:14:26,578 DEBUG [RS:1;448ea6c54a8c:35099 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:26,578 DEBUG [RS:2;448ea6c54a8c:35491 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:26,578 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '448ea6c54a8c,35099,1732101264437' 2024-11-20T11:14:26,578 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '448ea6c54a8c,35491,1732101264505' 2024-11-20T11:14:26,578 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T11:14:26,578 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T11:14:26,579 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:26,580 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(1482): Serving as 448ea6c54a8c,43507,1732101264298, RpcServer on 448ea6c54a8c/172.17.0.2:43507, sessionid=0x1013515b6630001 2024-11-20T11:14:26,580 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T11:14:26,580 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T11:14:26,580 DEBUG [RS:0;448ea6c54a8c:43507 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:26,580 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '448ea6c54a8c,43507,1732101264298' 2024-11-20T11:14:26,580 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T11:14:26,581 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T11:14:26,581 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T11:14:26,582 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T11:14:26,582 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T11:14:26,582 DEBUG [RS:2;448ea6c54a8c:35491 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:26,582 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '448ea6c54a8c,35491,1732101264505' 2024-11-20T11:14:26,582 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T11:14:26,582 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T11:14:26,582 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T11:14:26,582 DEBUG [RS:0;448ea6c54a8c:43507 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:26,582 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '448ea6c54a8c,43507,1732101264298' 2024-11-20T11:14:26,582 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T11:14:26,583 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T11:14:26,583 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T11:14:26,583 DEBUG [RS:1;448ea6c54a8c:35099 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:26,583 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '448ea6c54a8c,35099,1732101264437' 2024-11-20T11:14:26,583 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T11:14:26,583 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T11:14:26,584 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T11:14:26,585 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T11:14:26,585 DEBUG [RS:2;448ea6c54a8c:35491 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T11:14:26,585 INFO [RS:2;448ea6c54a8c:35491 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T11:14:26,585 INFO [RS:2;448ea6c54a8c:35491 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T11:14:26,586 DEBUG [RS:0;448ea6c54a8c:43507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T11:14:26,586 INFO [RS:0;448ea6c54a8c:43507 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T11:14:26,586 INFO [RS:0;448ea6c54a8c:43507 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T11:14:26,588 DEBUG [RS:1;448ea6c54a8c:35099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T11:14:26,588 INFO [RS:1;448ea6c54a8c:35099 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T11:14:26,588 INFO [RS:1;448ea6c54a8c:35099 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T11:14:26,692 INFO [RS:0;448ea6c54a8c:43507 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T11:14:26,692 INFO [RS:1;448ea6c54a8c:35099 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T11:14:26,693 INFO [RS:2;448ea6c54a8c:35491 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T11:14:26,696 INFO [RS:0;448ea6c54a8c:43507 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=448ea6c54a8c%2C43507%2C1732101264298, suffix=, logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,43507,1732101264298, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs, maxLogs=32 2024-11-20T11:14:26,697 INFO [RS:1;448ea6c54a8c:35099 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=448ea6c54a8c%2C35099%2C1732101264437, suffix=, logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35099,1732101264437, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs, maxLogs=32 2024-11-20T11:14:26,697 INFO [RS:2;448ea6c54a8c:35491 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=448ea6c54a8c%2C35491%2C1732101264505, suffix=, logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs, maxLogs=32 2024-11-20T11:14:26,723 DEBUG [RS:2;448ea6c54a8c:35491 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505/448ea6c54a8c%2C35491%2C1732101264505.1732101266706, exclude list is [], retry=0 2024-11-20T11:14:26,724 DEBUG [RS:0;448ea6c54a8c:43507 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,43507,1732101264298/448ea6c54a8c%2C43507%2C1732101264298.1732101266704, exclude list is [], retry=0 2024-11-20T11:14:26,729 WARN [IPC Server handler 0 on default port 35273 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T11:14:26,729 WARN [IPC Server handler 0 on default port 35273 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T11:14:26,730 WARN [IPC Server handler 0 on default port 35273 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T11:14:26,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37351,DS-dfc6e092-f541-494b-81e4-826b95dc8a56,DISK] 2024-11-20T11:14:26,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42077,DS-ef273f42-ddeb-44b0-90ec-87c34e9d8141,DISK] 2024-11-20T11:14:26,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44787,DS-5f4458a9-3703-4eaa-8233-a851e4aaf890,DISK] 2024-11-20T11:14:26,731 DEBUG [RS:1;448ea6c54a8c:35099 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35099,1732101264437/448ea6c54a8c%2C35099%2C1732101264437.1732101266705, exclude list is [], retry=0 2024-11-20T11:14:26,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37351,DS-dfc6e092-f541-494b-81e4-826b95dc8a56,DISK] 2024-11-20T11:14:26,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42077,DS-ef273f42-ddeb-44b0-90ec-87c34e9d8141,DISK] 2024-11-20T11:14:26,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37351,DS-dfc6e092-f541-494b-81e4-826b95dc8a56,DISK] 2024-11-20T11:14:26,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44787,DS-5f4458a9-3703-4eaa-8233-a851e4aaf890,DISK] 2024-11-20T11:14:26,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42077,DS-ef273f42-ddeb-44b0-90ec-87c34e9d8141,DISK] 2024-11-20T11:14:26,745 INFO [RS:2;448ea6c54a8c:35491 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505/448ea6c54a8c%2C35491%2C1732101264505.1732101266706 2024-11-20T11:14:26,752 DEBUG [RS:2;448ea6c54a8c:35491 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44047:44047),(127.0.0.1/127.0.0.1:40685:40685),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T11:14:26,757 INFO [RS:1;448ea6c54a8c:35099 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35099,1732101264437/448ea6c54a8c%2C35099%2C1732101264437.1732101266705 2024-11-20T11:14:26,760 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T11:14:26,762 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:26,763 INFO [RS:0;448ea6c54a8c:43507 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,43507,1732101264298/448ea6c54a8c%2C43507%2C1732101264298.1732101266704 2024-11-20T11:14:26,762 DEBUG [RS:1;448ea6c54a8c:35099 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44047:44047),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40685:40685)] 2024-11-20T11:14:26,765 DEBUG [RS:0;448ea6c54a8c:43507 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40685:40685)] 2024-11-20T11:14:26,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741835_1011 (size=32) 2024-11-20T11:14:26,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741835_1011 (size=32) 2024-11-20T11:14:26,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741835_1011 (size=32) 2024-11-20T11:14:26,801 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:26,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T11:14:26,807 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T11:14:26,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:26,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:26,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T11:14:26,811 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T11:14:26,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:26,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:26,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T11:14:26,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T11:14:26,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:26,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:26,819 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T11:14:26,829 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T11:14:26,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:26,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:26,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T11:14:26,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740 2024-11-20T11:14:26,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740 2024-11-20T11:14:26,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T11:14:26,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T11:14:26,838 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T11:14:26,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T11:14:26,845 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:26,846 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72003571, jitterRate=0.07293681800365448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T11:14:26,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732101266801Initializing all the Stores at 1732101266804 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101266804Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101266804Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101266804Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101266804Cleaning up temporary data from old regions at 1732101266837 (+33 ms)Region opened successfully at 1732101266848 (+11 ms) 2024-11-20T11:14:26,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T11:14:26,848 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T11:14:26,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T11:14:26,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T11:14:26,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T11:14:26,849 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T11:14:26,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732101266848Disabling compacts and flushes for region at 1732101266848Disabling writes for close at 1732101266848Writing region close event to WAL at 1732101266849 (+1 ms)Closed at 1732101266849 2024-11-20T11:14:26,853 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:26,853 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T11:14:26,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T11:14:26,876 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:26,880 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T11:14:27,033 DEBUG [448ea6c54a8c:38459 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T11:14:27,042 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(204): Hosts are {448ea6c54a8c=0} racks are {/default-rack=0} 2024-11-20T11:14:27,050 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T11:14:27,050 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T11:14:27,050 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T11:14:27,050 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T11:14:27,051 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T11:14:27,051 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T11:14:27,051 INFO [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T11:14:27,051 INFO [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T11:14:27,051 INFO [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T11:14:27,051 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T11:14:27,063 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:27,077 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,35491,1732101264505, state=OPENING 2024-11-20T11:14:27,088 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T11:14:27,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:27,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:27,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:27,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:27,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:27,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:27,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:27,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:27,096 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:27,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:27,281 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T11:14:27,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35243, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T11:14:27,306 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T11:14:27,306 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T11:14:27,307 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T11:14:27,311 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=448ea6c54a8c%2C35491%2C1732101264505.meta, suffix=.meta, logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs, maxLogs=32 2024-11-20T11:14:27,337 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, exclude list is [], retry=0 2024-11-20T11:14:27,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44787,DS-5f4458a9-3703-4eaa-8233-a851e4aaf890,DISK] 2024-11-20T11:14:27,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42077,DS-ef273f42-ddeb-44b0-90ec-87c34e9d8141,DISK] 2024-11-20T11:14:27,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37351,DS-dfc6e092-f541-494b-81e4-826b95dc8a56,DISK] 2024-11-20T11:14:27,356 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta 2024-11-20T11:14:27,356 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44047:44047),(127.0.0.1/127.0.0.1:40685:40685),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T11:14:27,357 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T11:14:27,359 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T11:14:27,362 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T11:14:27,369 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T11:14:27,374 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T11:14:27,375 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:27,375 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T11:14:27,376 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T11:14:27,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T11:14:27,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T11:14:27,383 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:27,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:27,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T11:14:27,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T11:14:27,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:27,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:27,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T11:14:27,390 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T11:14:27,390 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:27,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:27,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T11:14:27,393 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T11:14:27,393 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:27,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:27,394 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T11:14:27,395 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740 2024-11-20T11:14:27,401 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740 2024-11-20T11:14:27,405 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T11:14:27,405 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T11:14:27,407 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T11:14:27,411 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T11:14:27,413 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68708569, jitterRate=0.023837462067604065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T11:14:27,413 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T11:14:27,415 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732101267376Writing region info on filesystem at 1732101267376Initializing all the Stores at 1732101267379 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101267379Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101267380 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101267380Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101267380Cleaning up temporary data from old regions at 1732101267405 (+25 ms)Running coprocessor post-open hooks at 1732101267413 (+8 ms)Region opened successfully at 1732101267415 (+2 ms) 2024-11-20T11:14:27,425 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732101267271 2024-11-20T11:14:27,439 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T11:14:27,440 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T11:14:27,442 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:27,445 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,35491,1732101264505, state=OPEN 2024-11-20T11:14:27,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:27,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:27,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:27,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:27,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:27,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:27,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:27,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:27,449 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:27,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T11:14:27,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,35491,1732101264505 in 352 msec 2024-11-20T11:14:27,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T11:14:27,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 602 msec 2024-11-20T11:14:27,471 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:27,471 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T11:14:27,495 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T11:14:27,497 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1] 2024-11-20T11:14:27,528 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T11:14:27,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47195, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T11:14:27,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4710 sec 2024-11-20T11:14:27,558 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732101267558, completionTime=-1 2024-11-20T11:14:27,562 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T11:14:27,562 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:27,591 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-20T11:14:27,591 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101327591 2024-11-20T11:14:27,592 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101387592 2024-11-20T11:14:27,592 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 29 msec 2024-11-20T11:14:27,593 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:27,602 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,38459,1732101263372-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:27,602 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,38459,1732101263372-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:27,602 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,38459,1732101263372-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:27,604 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-448ea6c54a8c:38459, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:27,605 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:27,608 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:27,612 DEBUG [master/448ea6c54a8c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T11:14:27,644 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.011sec 2024-11-20T11:14:27,650 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T11:14:27,651 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T11:14:27,652 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T11:14:27,653 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T11:14:27,653 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T11:14:27,654 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,38459,1732101263372-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T11:14:27,654 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,38459,1732101263372-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T11:14:27,663 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T11:14:27,663 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T11:14:27,664 INFO [master/448ea6c54a8c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=448ea6c54a8c,38459,1732101263372-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T11:14:27,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd5940a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T11:14:27,685 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T11:14:27,685 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T11:14:27,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 448ea6c54a8c,38459,-1 for getting cluster id 2024-11-20T11:14:27,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T11:14:27,702 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '486e3fde-64a7-493d-99c8-877072beb5df' 2024-11-20T11:14:27,705 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T11:14:27,705 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "486e3fde-64a7-493d-99c8-877072beb5df" 2024-11-20T11:14:27,706 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2278a859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T11:14:27,706 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [448ea6c54a8c,38459,-1] 2024-11-20T11:14:27,708 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T11:14:27,712 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:27,712 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T11:14:27,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a203d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T11:14:27,716 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T11:14:27,726 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1] 2024-11-20T11:14:27,727 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T11:14:27,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T11:14:27,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:27,766 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T11:14:27,778 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:27,782 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65383b59 2024-11-20T11:14:27,783 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T11:14:27,799 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T11:14:27,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T11:14:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:27,874 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T11:14:27,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-20T11:14:27,883 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:27,886 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T11:14:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T11:14:27,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741837_1013 (size=358) 2024-11-20T11:14:27,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741837_1013 (size=358) 2024-11-20T11:14:27,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741837_1013 (size=358) 2024-11-20T11:14:27,921 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e9ad74913c92386ef2a8194575ccf6d, NAME => 'test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:27,921 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0c331e73b55d2ae38913c70b5317c877, NAME => 'test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:27,922 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ab39ce583e7145069013a6bec41b40c0, NAME => 'test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:27,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741838_1014 (size=45) 2024-11-20T11:14:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741838_1014 (size=45) 2024-11-20T11:14:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741838_1014 (size=45) 2024-11-20T11:14:27,945 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:27,945 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ab39ce583e7145069013a6bec41b40c0, disabling compactions & flushes 2024-11-20T11:14:27,945 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:27,945 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:27,946 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. after waiting 0 ms 2024-11-20T11:14:27,946 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:27,946 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:27,946 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ab39ce583e7145069013a6bec41b40c0: Waiting for close lock at 1732101267945Disabling compacts and flushes for region at 1732101267945Disabling writes for close at 1732101267946 (+1 ms)Writing region close event to WAL at 1732101267946Closed at 1732101267946 2024-11-20T11:14:27,947 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => cb4960297e33c5789bed6704818bf798, NAME => 'test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:27,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741839_1015 (size=42) 2024-11-20T11:14:27,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741839_1015 (size=42) 2024-11-20T11:14:27,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741839_1015 (size=42) 2024-11-20T11:14:27,964 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:27,967 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1e9ad74913c92386ef2a8194575ccf6d, disabling compactions & flushes 2024-11-20T11:14:27,967 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:27,967 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:27,967 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. after waiting 0 ms 2024-11-20T11:14:27,967 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:27,967 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:27,967 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e9ad74913c92386ef2a8194575ccf6d: Waiting for close lock at 1732101267964Disabling compacts and flushes for region at 1732101267964Disabling writes for close at 1732101267967 (+3 ms)Writing region close event to WAL at 1732101267967Closed at 1732101267967 2024-11-20T11:14:27,968 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bac309a4ff803986756ba76688d03707, NAME => 'test,eee,1732101267801.bac309a4ff803986756ba76688d03707.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:27,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741840_1016 (size=45) 2024-11-20T11:14:27,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741840_1016 (size=45) 2024-11-20T11:14:27,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741840_1016 (size=45) 2024-11-20T11:14:27,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:27,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0c331e73b55d2ae38913c70b5317c877, disabling compactions & flushes 2024-11-20T11:14:27,989 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877. 2024-11-20T11:14:27,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877. 2024-11-20T11:14:27,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877. after waiting 0 ms 2024-11-20T11:14:27,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877. 2024-11-20T11:14:27,989 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877. 2024-11-20T11:14:27,989 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0c331e73b55d2ae38913c70b5317c877: Waiting for close lock at 1732101267989Disabling compacts and flushes for region at 1732101267989Disabling writes for close at 1732101267989Writing region close event to WAL at 1732101267989Closed at 1732101267989 2024-11-20T11:14:27,990 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1a244f5f504c74b0b1dff6963a37aff0, NAME => 'test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T11:14:28,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741841_1017 (size=45) 2024-11-20T11:14:28,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741841_1017 (size=45) 2024-11-20T11:14:28,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741841_1017 (size=45) 2024-11-20T11:14:28,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,034 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing cb4960297e33c5789bed6704818bf798, disabling compactions & flushes 2024-11-20T11:14:28,035 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:28,035 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:28,035 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. after waiting 0 ms 2024-11-20T11:14:28,035 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:28,035 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:28,035 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for cb4960297e33c5789bed6704818bf798: Waiting for close lock at 1732101268034Disabling compacts and flushes for region at 1732101268034Disabling writes for close at 1732101268035 (+1 ms)Writing region close event to WAL at 1732101268035Closed at 1732101268035 2024-11-20T11:14:28,036 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 19c7dd4cb6560dd941c762414bfe9afa, NAME => 'test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741842_1018 (size=45) 2024-11-20T11:14:28,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741842_1018 (size=45) 2024-11-20T11:14:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741842_1018 (size=45) 2024-11-20T11:14:28,079 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1732101267801.bac309a4ff803986756ba76688d03707.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,079 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bac309a4ff803986756ba76688d03707, disabling compactions & flushes 2024-11-20T11:14:28,079 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:28,080 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:28,080 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732101267801.bac309a4ff803986756ba76688d03707. after waiting 0 ms 2024-11-20T11:14:28,080 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:28,080 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:28,080 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bac309a4ff803986756ba76688d03707: Waiting for close lock at 1732101268079Disabling compacts and flushes for region at 1732101268079Disabling writes for close at 1732101268080 (+1 ms)Writing region close event to WAL at 1732101268080Closed at 1732101268080 2024-11-20T11:14:28,081 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ddf1f8c8ecace91d120dbb68cbf18fb1, NAME => 'test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741843_1019 (size=45) 2024-11-20T11:14:28,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741843_1019 (size=45) 2024-11-20T11:14:28,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741843_1019 (size=45) 2024-11-20T11:14:28,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1a244f5f504c74b0b1dff6963a37aff0, disabling compactions & flushes 2024-11-20T11:14:28,130 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:28,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:28,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. after waiting 0 ms 2024-11-20T11:14:28,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:28,130 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:28,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1a244f5f504c74b0b1dff6963a37aff0: Waiting for close lock at 1732101268130Disabling compacts and flushes for region at 1732101268130Disabling writes for close at 1732101268130Writing region close event to WAL at 1732101268130Closed at 1732101268130 2024-11-20T11:14:28,131 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4e8a8778f20ffb0d01039b06a807ee6a, NAME => 'test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741844_1020 (size=45) 2024-11-20T11:14:28,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741844_1020 (size=45) 2024-11-20T11:14:28,150 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 19c7dd4cb6560dd941c762414bfe9afa, disabling compactions & flushes 2024-11-20T11:14:28,151 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:28,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:28,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. after waiting 0 ms 2024-11-20T11:14:28,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:28,151 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:28,151 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 19c7dd4cb6560dd941c762414bfe9afa: Waiting for close lock at 1732101268151Disabling compacts and flushes for region at 1732101268151Disabling writes for close at 1732101268151Writing region close event to WAL at 1732101268151Closed at 1732101268151 2024-11-20T11:14:28,152 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 073880d856b42f096b9b41dfd0b349af, NAME => 'test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741844_1020 (size=45) 2024-11-20T11:14:28,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741845_1021 (size=45) 2024-11-20T11:14:28,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741845_1021 (size=45) 2024-11-20T11:14:28,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741845_1021 (size=45) 2024-11-20T11:14:28,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741847_1023 (size=45) 2024-11-20T11:14:28,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ddf1f8c8ecace91d120dbb68cbf18fb1, disabling compactions & flushes 2024-11-20T11:14:28,213 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:28,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:28,214 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. after waiting 0 ms 2024-11-20T11:14:28,214 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:28,214 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:28,214 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ddf1f8c8ecace91d120dbb68cbf18fb1: Waiting for close lock at 1732101268213Disabling compacts and flushes for region at 1732101268213Disabling writes for close at 1732101268214 (+1 ms)Writing region close event to WAL at 1732101268214Closed at 1732101268214 2024-11-20T11:14:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T11:14:28,215 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 70c2f19bb4361dae17cea153d84e898a, NAME => 'test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741847_1023 (size=45) 2024-11-20T11:14:28,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741847_1023 (size=45) 2024-11-20T11:14:28,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 073880d856b42f096b9b41dfd0b349af, disabling compactions & flushes 2024-11-20T11:14:28,224 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. 2024-11-20T11:14:28,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. 2024-11-20T11:14:28,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. after waiting 0 ms 2024-11-20T11:14:28,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. 2024-11-20T11:14:28,224 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. 2024-11-20T11:14:28,224 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 073880d856b42f096b9b41dfd0b349af: Waiting for close lock at 1732101268224Disabling compacts and flushes for region at 1732101268224Disabling writes for close at 1732101268224Writing region close event to WAL at 1732101268224Closed at 1732101268224 2024-11-20T11:14:28,225 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d221d096301219259fde4786f6bb7402, NAME => 'test,lll,1732101267801.d221d096301219259fde4786f6bb7402.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741846_1022 (size=45) 2024-11-20T11:14:28,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741846_1022 (size=45) 2024-11-20T11:14:28,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741846_1022 (size=45) 2024-11-20T11:14:28,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4e8a8778f20ffb0d01039b06a807ee6a, disabling compactions & flushes 2024-11-20T11:14:28,231 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:28,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:28,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. after waiting 0 ms 2024-11-20T11:14:28,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:28,231 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:28,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4e8a8778f20ffb0d01039b06a807ee6a: Waiting for close lock at 1732101268231Disabling compacts and flushes for region at 1732101268231Disabling writes for close at 1732101268231Writing region close event to WAL at 1732101268231Closed at 1732101268231 2024-11-20T11:14:28,232 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 94ab7bd7c2c2041d4db8eb8309fa9cea, NAME => 'test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741848_1024 (size=45) 2024-11-20T11:14:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741848_1024 (size=45) 2024-11-20T11:14:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741848_1024 (size=45) 2024-11-20T11:14:28,279 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 70c2f19bb4361dae17cea153d84e898a, disabling compactions & flushes 2024-11-20T11:14:28,285 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:28,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:28,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. after waiting 0 ms 2024-11-20T11:14:28,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:28,285 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:28,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 70c2f19bb4361dae17cea153d84e898a: Waiting for close lock at 1732101268285Disabling compacts and flushes for region at 1732101268285Disabling writes for close at 1732101268285Writing region close event to WAL at 1732101268285Closed at 1732101268285 2024-11-20T11:14:28,287 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c8b0425cde259974fdc42bd0afa1e9d6, NAME => 'test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741849_1025 (size=45) 2024-11-20T11:14:28,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741849_1025 (size=45) 2024-11-20T11:14:28,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741849_1025 (size=45) 2024-11-20T11:14:28,336 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1732101267801.d221d096301219259fde4786f6bb7402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,336 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d221d096301219259fde4786f6bb7402, disabling compactions & flushes 2024-11-20T11:14:28,336 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:28,336 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:28,336 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732101267801.d221d096301219259fde4786f6bb7402. after waiting 0 ms 2024-11-20T11:14:28,337 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:28,337 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:28,337 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d221d096301219259fde4786f6bb7402: Waiting for close lock at 1732101268336Disabling compacts and flushes for region at 1732101268336Disabling writes for close at 1732101268336Writing region close event to WAL at 1732101268337 (+1 ms)Closed at 1732101268337 2024-11-20T11:14:28,338 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ec000da7633d9b0b0b955ae7aa7e962f, NAME => 'test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741850_1026 (size=45) 2024-11-20T11:14:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741850_1026 (size=45) 2024-11-20T11:14:28,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741850_1026 (size=45) 2024-11-20T11:14:28,347 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,348 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 94ab7bd7c2c2041d4db8eb8309fa9cea, disabling compactions & flushes 2024-11-20T11:14:28,348 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:28,348 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:28,348 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. after waiting 0 ms 2024-11-20T11:14:28,348 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:28,348 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:28,348 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 94ab7bd7c2c2041d4db8eb8309fa9cea: Waiting for close lock at 1732101268347Disabling compacts and flushes for region at 1732101268347Disabling writes for close at 1732101268348 (+1 ms)Writing region close event to WAL at 1732101268348Closed at 1732101268348 2024-11-20T11:14:28,349 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 031ddda36eb9bb40423f7c9f4322100c, NAME => 'test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741853_1029 (size=45) 2024-11-20T11:14:28,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741853_1029 (size=45) 2024-11-20T11:14:28,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741853_1029 (size=45) 2024-11-20T11:14:28,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741852_1028 (size=45) 2024-11-20T11:14:28,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741852_1028 (size=45) 2024-11-20T11:14:28,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741852_1028 (size=45) 2024-11-20T11:14:28,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ec000da7633d9b0b0b955ae7aa7e962f, disabling compactions & flushes 2024-11-20T11:14:28,474 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:28,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:28,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. after waiting 0 ms 2024-11-20T11:14:28,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:28,474 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:28,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ec000da7633d9b0b0b955ae7aa7e962f: Waiting for close lock at 1732101268474Disabling compacts and flushes for region at 1732101268474Disabling writes for close at 1732101268474Writing region close event to WAL at 1732101268474Closed at 1732101268474 2024-11-20T11:14:28,475 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 171d75c7056fb7f3c57a7e0ca3724f6d, NAME => 'test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741851_1027 (size=45) 2024-11-20T11:14:28,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741851_1027 (size=45) 2024-11-20T11:14:28,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741851_1027 (size=45) 2024-11-20T11:14:28,501 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c8b0425cde259974fdc42bd0afa1e9d6, disabling compactions & flushes 2024-11-20T11:14:28,502 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6. 2024-11-20T11:14:28,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6. 2024-11-20T11:14:28,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6. after waiting 0 ms 2024-11-20T11:14:28,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6. 2024-11-20T11:14:28,502 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6. 2024-11-20T11:14:28,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c8b0425cde259974fdc42bd0afa1e9d6: Waiting for close lock at 1732101268502Disabling compacts and flushes for region at 1732101268502Disabling writes for close at 1732101268502Writing region close event to WAL at 1732101268502Closed at 1732101268502 2024-11-20T11:14:28,503 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6864708d24f9dd54d37dd8e832ca6cb9, NAME => 'test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T11:14:28,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741855_1031 (size=45) 2024-11-20T11:14:28,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741855_1031 (size=45) 2024-11-20T11:14:28,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741855_1031 (size=45) 2024-11-20T11:14:28,552 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,553 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6864708d24f9dd54d37dd8e832ca6cb9, disabling compactions & flushes 2024-11-20T11:14:28,553 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:28,553 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:28,553 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. after waiting 0 ms 2024-11-20T11:14:28,553 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:28,553 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:28,553 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6864708d24f9dd54d37dd8e832ca6cb9: Waiting for close lock at 1732101268553Disabling compacts and flushes for region at 1732101268553Disabling writes for close at 1732101268553Writing region close event to WAL at 1732101268553Closed at 1732101268553 2024-11-20T11:14:28,554 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 10e6b7dac8e72b3715b09f737ea5dce3, NAME => 'test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741854_1030 (size=45) 2024-11-20T11:14:28,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741854_1030 (size=45) 2024-11-20T11:14:28,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741854_1030 (size=45) 2024-11-20T11:14:28,572 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,572 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 171d75c7056fb7f3c57a7e0ca3724f6d, disabling compactions & flushes 2024-11-20T11:14:28,572 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:28,572 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:28,572 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. after waiting 0 ms 2024-11-20T11:14:28,572 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:28,572 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:28,573 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 171d75c7056fb7f3c57a7e0ca3724f6d: Waiting for close lock at 1732101268572Disabling compacts and flushes for region at 1732101268572Disabling writes for close at 1732101268572Writing region close event to WAL at 1732101268572Closed at 1732101268572 2024-11-20T11:14:28,573 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 07feff3ab7eab5622741a0364ece2464, NAME => 'test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741856_1032 (size=45) 2024-11-20T11:14:28,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741856_1032 (size=45) 2024-11-20T11:14:28,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741856_1032 (size=45) 2024-11-20T11:14:28,630 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,630 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 10e6b7dac8e72b3715b09f737ea5dce3, disabling compactions & flushes 2024-11-20T11:14:28,630 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:28,630 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:28,631 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. after waiting 0 ms 2024-11-20T11:14:28,631 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:28,631 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:28,631 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 10e6b7dac8e72b3715b09f737ea5dce3: Waiting for close lock at 1732101268630Disabling compacts and flushes for region at 1732101268630Disabling writes for close at 1732101268631 (+1 ms)Writing region close event to WAL at 1732101268631Closed at 1732101268631 2024-11-20T11:14:28,632 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 16717e829fb199e0c26298a5116a0b4e, NAME => 'test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741857_1033 (size=45) 2024-11-20T11:14:28,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741857_1033 (size=45) 2024-11-20T11:14:28,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741857_1033 (size=45) 2024-11-20T11:14:28,672 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,673 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 07feff3ab7eab5622741a0364ece2464, disabling compactions & flushes 2024-11-20T11:14:28,673 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:28,673 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:28,673 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. after waiting 0 ms 2024-11-20T11:14:28,673 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:28,673 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:28,673 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 07feff3ab7eab5622741a0364ece2464: Waiting for close lock at 1732101268673Disabling compacts and flushes for region at 1732101268673Disabling writes for close at 1732101268673Writing region close event to WAL at 1732101268673Closed at 1732101268673 2024-11-20T11:14:28,674 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2b5da0de9b95a9822f9526f93d4d7af1, NAME => 'test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741858_1034 (size=45) 2024-11-20T11:14:28,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741858_1034 (size=45) 2024-11-20T11:14:28,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741858_1034 (size=45) 2024-11-20T11:14:28,697 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,697 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 16717e829fb199e0c26298a5116a0b4e, disabling compactions & flushes 2024-11-20T11:14:28,697 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:28,697 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:28,697 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. after waiting 0 ms 2024-11-20T11:14:28,697 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:28,698 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:28,698 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 16717e829fb199e0c26298a5116a0b4e: Waiting for close lock at 1732101268697Disabling compacts and flushes for region at 1732101268697Disabling writes for close at 1732101268697Writing region close event to WAL at 1732101268697Closed at 1732101268697 2024-11-20T11:14:28,698 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 349e25c47fdd7d8457f465670572f372, NAME => 'test,www,1732101267801.349e25c47fdd7d8457f465670572f372.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741859_1035 (size=45) 2024-11-20T11:14:28,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741859_1035 (size=45) 2024-11-20T11:14:28,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741859_1035 (size=45) 2024-11-20T11:14:28,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2b5da0de9b95a9822f9526f93d4d7af1, disabling compactions & flushes 2024-11-20T11:14:28,750 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1. 2024-11-20T11:14:28,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1. 2024-11-20T11:14:28,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1. after waiting 0 ms 2024-11-20T11:14:28,750 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1. 2024-11-20T11:14:28,750 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1. 2024-11-20T11:14:28,751 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2b5da0de9b95a9822f9526f93d4d7af1: Waiting for close lock at 1732101268750Disabling compacts and flushes for region at 1732101268750Disabling writes for close at 1732101268750Writing region close event to WAL at 1732101268750Closed at 1732101268750 2024-11-20T11:14:28,752 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 400f5d1df45ccf5378a59a8cd78bb9d8, NAME => 'test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741860_1036 (size=45) 2024-11-20T11:14:28,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741860_1036 (size=45) 2024-11-20T11:14:28,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741860_1036 (size=45) 2024-11-20T11:14:28,787 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732101267801.349e25c47fdd7d8457f465670572f372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,787 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 349e25c47fdd7d8457f465670572f372, disabling compactions & flushes 2024-11-20T11:14:28,787 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:28,787 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:28,787 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732101267801.349e25c47fdd7d8457f465670572f372. after waiting 0 ms 2024-11-20T11:14:28,787 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:28,787 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:28,787 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 349e25c47fdd7d8457f465670572f372: Waiting for close lock at 1732101268787Disabling compacts and flushes for region at 1732101268787Disabling writes for close at 1732101268787Writing region close event to WAL at 1732101268787Closed at 1732101268787 2024-11-20T11:14:28,788 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 53a2311878635447fc5975caced0a7d4, NAME => 'test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 031ddda36eb9bb40423f7c9f4322100c, disabling compactions & flushes 2024-11-20T11:14:28,845 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:28,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:28,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. after waiting 0 ms 2024-11-20T11:14:28,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:28,845 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:28,845 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 031ddda36eb9bb40423f7c9f4322100c: Waiting for close lock at 1732101268845Disabling compacts and flushes for region at 1732101268845Disabling writes for close at 1732101268845Writing region close event to WAL at 1732101268845Closed at 1732101268845 2024-11-20T11:14:28,846 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fb17647ae131913beb1233c85f78d4dc, NAME => 'test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a 2024-11-20T11:14:28,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741861_1037 (size=45) 2024-11-20T11:14:28,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741861_1037 (size=45) 2024-11-20T11:14:28,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741861_1037 (size=45) 2024-11-20T11:14:28,868 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,869 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 400f5d1df45ccf5378a59a8cd78bb9d8, disabling compactions & flushes 2024-11-20T11:14:28,869 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:28,869 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:28,869 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. after waiting 0 ms 2024-11-20T11:14:28,869 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:28,869 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:28,869 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 400f5d1df45ccf5378a59a8cd78bb9d8: Waiting for close lock at 1732101268869Disabling compacts and flushes for region at 1732101268869Disabling writes for close at 1732101268869Writing region close event to WAL at 1732101268869Closed at 1732101268869 2024-11-20T11:14:28,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741862_1038 (size=45) 2024-11-20T11:14:28,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741862_1038 (size=45) 2024-11-20T11:14:28,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741862_1038 (size=45) 2024-11-20T11:14:28,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 53a2311878635447fc5975caced0a7d4, disabling compactions & flushes 2024-11-20T11:14:28,886 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:28,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:28,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. after waiting 0 ms 2024-11-20T11:14:28,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:28,886 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:28,887 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 53a2311878635447fc5975caced0a7d4: Waiting for close lock at 1732101268886Disabling compacts and flushes for region at 1732101268886Disabling writes for close at 1732101268886Writing region close event to WAL at 1732101268886Closed at 1732101268886 2024-11-20T11:14:28,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741863_1039 (size=42) 2024-11-20T11:14:28,913 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:28,913 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fb17647ae131913beb1233c85f78d4dc, disabling compactions & flushes 2024-11-20T11:14:28,913 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:28,913 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:28,913 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. after waiting 0 ms 2024-11-20T11:14:28,913 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:28,913 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:28,913 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fb17647ae131913beb1233c85f78d4dc: Waiting for close lock at 1732101268913Disabling compacts and flushes for region at 1732101268913Disabling writes for close at 1732101268913Writing region close event to WAL at 1732101268913Closed at 1732101268913 2024-11-20T11:14:28,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741863_1039 (size=42) 2024-11-20T11:14:28,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741863_1039 (size=42) 2024-11-20T11:14:28,917 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T11:14:28,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732101267801.bac309a4ff803986756ba76688d03707.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732101267801.d221d096301219259fde4786f6bb7402.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,931 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,931 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,931 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,931 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732101267801.349e25c47fdd7d8457f465670572f372.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:28,934 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732101268917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732101268917"}]},"ts":"1732101268917"} 2024-11-20T11:14:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T11:14:29,033 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-20T11:14:29,037 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T11:14:29,042 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732101269038"}]},"ts":"1732101269038"} 2024-11-20T11:14:29,059 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-20T11:14:29,060 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {448ea6c54a8c=0} racks are {/default-rack=0} 2024-11-20T11:14:29,104 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T11:14:29,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T11:14:29,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T11:14:29,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T11:14:29,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T11:14:29,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T11:14:29,105 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T11:14:29,105 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T11:14:29,105 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T11:14:29,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T11:14:29,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e9ad74913c92386ef2a8194575ccf6d, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab39ce583e7145069013a6bec41b40c0, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cb4960297e33c5789bed6704818bf798, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bac309a4ff803986756ba76688d03707, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=19c7dd4cb6560dd941c762414bfe9afa, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ddf1f8c8ecace91d120dbb68cbf18fb1, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e8a8778f20ffb0d01039b06a807ee6a, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=70c2f19bb4361dae17cea153d84e898a, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=94ab7bd7c2c2041d4db8eb8309fa9cea, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ec000da7633d9b0b0b955ae7aa7e962f, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=031ddda36eb9bb40423f7c9f4322100c, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=171d75c7056fb7f3c57a7e0ca3724f6d, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6864708d24f9dd54d37dd8e832ca6cb9, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07feff3ab7eab5622741a0364ece2464, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=16717e829fb199e0c26298a5116a0b4e, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=349e25c47fdd7d8457f465670572f372, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=53a2311878635447fc5975caced0a7d4, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fb17647ae131913beb1233c85f78d4dc, ASSIGN}] 2024-11-20T11:14:29,143 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fb17647ae131913beb1233c85f78d4dc, ASSIGN 2024-11-20T11:14:29,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fb17647ae131913beb1233c85f78d4dc, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:29,150 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=53a2311878635447fc5975caced0a7d4, ASSIGN 2024-11-20T11:14:29,152 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=53a2311878635447fc5975caced0a7d4, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:29,154 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:29,157 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35491,1732101264505; forceNewPlan=false, retain=false 2024-11-20T11:14:29,160 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=349e25c47fdd7d8457f465670572f372, ASSIGN 2024-11-20T11:14:29,162 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=349e25c47fdd7d8457f465670572f372, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:29,165 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:29,167 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35491,1732101264505; forceNewPlan=false, retain=false 2024-11-20T11:14:29,170 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=16717e829fb199e0c26298a5116a0b4e, ASSIGN 2024-11-20T11:14:29,175 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=16717e829fb199e0c26298a5116a0b4e, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:29,178 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07feff3ab7eab5622741a0364ece2464, ASSIGN 2024-11-20T11:14:29,181 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=07feff3ab7eab5622741a0364ece2464, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:29,189 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:29,194 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35491,1732101264505; forceNewPlan=false, retain=false 2024-11-20T11:14:29,199 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6864708d24f9dd54d37dd8e832ca6cb9, ASSIGN 2024-11-20T11:14:29,209 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6864708d24f9dd54d37dd8e832ca6cb9, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:29,215 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=171d75c7056fb7f3c57a7e0ca3724f6d, ASSIGN 2024-11-20T11:14:29,219 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=171d75c7056fb7f3c57a7e0ca3724f6d, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:29,222 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=031ddda36eb9bb40423f7c9f4322100c, ASSIGN 2024-11-20T11:14:29,228 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=031ddda36eb9bb40423f7c9f4322100c, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:29,234 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ec000da7633d9b0b0b955ae7aa7e962f, ASSIGN 2024-11-20T11:14:29,237 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ec000da7633d9b0b0b955ae7aa7e962f, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:29,240 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:29,253 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35491,1732101264505; forceNewPlan=false, retain=false 2024-11-20T11:14:29,255 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=94ab7bd7c2c2041d4db8eb8309fa9cea, ASSIGN 2024-11-20T11:14:29,260 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=94ab7bd7c2c2041d4db8eb8309fa9cea, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:29,265 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:29,267 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35491,1732101264505; forceNewPlan=false, retain=false 2024-11-20T11:14:29,269 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=70c2f19bb4361dae17cea153d84e898a, ASSIGN 2024-11-20T11:14:29,274 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=70c2f19bb4361dae17cea153d84e898a, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:29,277 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:29,278 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35491,1732101264505; forceNewPlan=false, retain=false 2024-11-20T11:14:29,280 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e8a8778f20ffb0d01039b06a807ee6a, ASSIGN 2024-11-20T11:14:29,283 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4e8a8778f20ffb0d01039b06a807ee6a, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:29,285 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ddf1f8c8ecace91d120dbb68cbf18fb1, ASSIGN 2024-11-20T11:14:29,293 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ddf1f8c8ecace91d120dbb68cbf18fb1, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:29,295 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=19c7dd4cb6560dd941c762414bfe9afa, ASSIGN 2024-11-20T11:14:29,297 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=19c7dd4cb6560dd941c762414bfe9afa, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:29,299 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:29,299 INFO [448ea6c54a8c:38459 {}] balancer.BaseLoadBalancer(388): Reassigned 19 regions. 19 retained the pre-restart assignment. 2024-11-20T11:14:29,302 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35491,1732101264505; forceNewPlan=false, retain=false 2024-11-20T11:14:29,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=53a2311878635447fc5975caced0a7d4, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,305 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=349e25c47fdd7d8457f465670572f372, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,306 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=07feff3ab7eab5622741a0364ece2464, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=53a2311878635447fc5975caced0a7d4, ASSIGN because future has completed 2024-11-20T11:14:29,312 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,314 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,316 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,317 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,317 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=d221d096301219259fde4786f6bb7402, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,320 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=fb17647ae131913beb1233c85f78d4dc, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,321 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=16717e829fb199e0c26298a5116a0b4e, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,322 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,322 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=031ddda36eb9bb40423f7c9f4322100c, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,323 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=70c2f19bb4361dae17cea153d84e898a, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,325 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bac309a4ff803986756ba76688d03707, ASSIGN 2024-11-20T11:14:29,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=349e25c47fdd7d8457f465670572f372, ASSIGN because future has completed 2024-11-20T11:14:29,347 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bac309a4ff803986756ba76688d03707, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:29,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53a2311878635447fc5975caced0a7d4, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:29,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=07feff3ab7eab5622741a0364ece2464, ASSIGN because future has completed 2024-11-20T11:14:29,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6864708d24f9dd54d37dd8e832ca6cb9, ASSIGN because future has completed 2024-11-20T11:14:29,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07feff3ab7eab5622741a0364ece2464, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:29,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=94ab7bd7c2c2041d4db8eb8309fa9cea, ASSIGN because future has completed 2024-11-20T11:14:29,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4e8a8778f20ffb0d01039b06a807ee6a, ASSIGN because future has completed 2024-11-20T11:14:29,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN because future has completed 2024-11-20T11:14:29,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN because future has completed 2024-11-20T11:14:29,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:29,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN because future has completed 2024-11-20T11:14:29,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN because future has completed 2024-11-20T11:14:29,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN because future has completed 2024-11-20T11:14:29,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN because future has completed 2024-11-20T11:14:29,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 073880d856b42f096b9b41dfd0b349af, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:29,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fb17647ae131913beb1233c85f78d4dc, ASSIGN because future has completed 2024-11-20T11:14:29,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=16717e829fb199e0c26298a5116a0b4e, ASSIGN because future has completed 2024-11-20T11:14:29,450 INFO [448ea6c54a8c:38459 {}] balancer.BaseLoadBalancer(388): Reassigned 3 regions. 3 retained the pre-restart assignment. 2024-11-20T11:14:29,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=171d75c7056fb7f3c57a7e0ca3724f6d, ASSIGN because future has completed 2024-11-20T11:14:29,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=031ddda36eb9bb40423f7c9f4322100c, ASSIGN because future has completed 2024-11-20T11:14:29,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ec000da7633d9b0b0b955ae7aa7e962f, ASSIGN because future has completed 2024-11-20T11:14:29,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 031ddda36eb9bb40423f7c9f4322100c, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:29,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=70c2f19bb4361dae17cea153d84e898a, ASSIGN because future has completed 2024-11-20T11:14:29,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ddf1f8c8ecace91d120dbb68cbf18fb1, ASSIGN because future has completed 2024-11-20T11:14:29,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddf1f8c8ecace91d120dbb68cbf18fb1, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:29,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70c2f19bb4361dae17cea153d84e898a, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:29,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec000da7633d9b0b0b955ae7aa7e962f, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:29,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 171d75c7056fb7f3c57a7e0ca3724f6d, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:29,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,504 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=bac309a4ff803986756ba76688d03707, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16717e829fb199e0c26298a5116a0b4e, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:29,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=19c7dd4cb6560dd941c762414bfe9afa, ASSIGN because future has completed 2024-11-20T11:14:29,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19c7dd4cb6560dd941c762414bfe9afa, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:29,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN because future has completed 2024-11-20T11:14:29,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bac309a4ff803986756ba76688d03707, ASSIGN because future has completed 2024-11-20T11:14:29,529 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T11:14:29,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure bac309a4ff803986756ba76688d03707, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:29,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38021, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T11:14:29,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:29,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb17647ae131913beb1233c85f78d4dc, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:29,609 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:29,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:29,609 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 07feff3ab7eab5622741a0364ece2464, NAME => 'test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-20T11:14:29,610 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,610 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,610 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,610 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,617 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T11:14:29,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:29,619 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. 2024-11-20T11:14:29,619 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 073880d856b42f096b9b41dfd0b349af, NAME => 'test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-20T11:14:29,620 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,620 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,620 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,620 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:29,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54205, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T11:14:29,637 INFO [StoreOpener-07feff3ab7eab5622741a0364ece2464-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:29,643 INFO [StoreOpener-073880d856b42f096b9b41dfd0b349af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,645 INFO [StoreOpener-07feff3ab7eab5622741a0364ece2464-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07feff3ab7eab5622741a0364ece2464 columnFamilyName family 2024-11-20T11:14:29,645 DEBUG [StoreOpener-07feff3ab7eab5622741a0364ece2464-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,646 INFO [StoreOpener-07feff3ab7eab5622741a0364ece2464-1 {}] regionserver.HStore(327): Store=07feff3ab7eab5622741a0364ece2464/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,646 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,648 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,650 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,651 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,651 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,653 INFO [StoreOpener-073880d856b42f096b9b41dfd0b349af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 073880d856b42f096b9b41dfd0b349af columnFamilyName family 2024-11-20T11:14:29,653 DEBUG [StoreOpener-073880d856b42f096b9b41dfd0b349af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,654 INFO [StoreOpener-073880d856b42f096b9b41dfd0b349af-1 {}] regionserver.HStore(327): Store=073880d856b42f096b9b41dfd0b349af/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,654 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,656 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,656 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,657 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,657 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,657 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,662 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:29,662 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => ec000da7633d9b0b0b955ae7aa7e962f, NAME => 'test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-20T11:14:29,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e8a8778f20ffb0d01039b06a807ee6a, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:29,662 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,662 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,663 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,663 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,663 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94ab7bd7c2c2041d4db8eb8309fa9cea, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:29,677 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/07feff3ab7eab5622741a0364ece2464/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6864708d24f9dd54d37dd8e832ca6cb9, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:29,678 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 07feff3ab7eab5622741a0364ece2464; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74384957, jitterRate=0.10842223465442657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,679 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:29,679 INFO [StoreOpener-ec000da7633d9b0b0b955ae7aa7e962f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,680 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 07feff3ab7eab5622741a0364ece2464: Running coprocessor pre-open hook at 1732101269611Writing region info on filesystem at 1732101269611Initializing all the Stores at 1732101269620 (+9 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269620Cleaning up temporary data from old regions at 1732101269651 (+31 ms)Running coprocessor post-open hooks at 1732101269679 (+28 ms)Region opened successfully at 1732101269680 (+1 ms) 2024-11-20T11:14:29,683 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464., pid=32, masterSystemTime=1732101269528 2024-11-20T11:14:29,685 INFO [StoreOpener-ec000da7633d9b0b0b955ae7aa7e962f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec000da7633d9b0b0b955ae7aa7e962f columnFamilyName family 2024-11-20T11:14:29,685 DEBUG [StoreOpener-ec000da7633d9b0b0b955ae7aa7e962f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,686 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:29,686 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:29,686 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:29,686 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 53a2311878635447fc5975caced0a7d4, NAME => 'test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-20T11:14:29,687 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,687 INFO [StoreOpener-ec000da7633d9b0b0b955ae7aa7e962f-1 {}] regionserver.HStore(327): Store=ec000da7633d9b0b0b955ae7aa7e962f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,687 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,688 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,688 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,688 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,689 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,690 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/073880d856b42f096b9b41dfd0b349af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,690 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,691 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,691 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,691 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 073880d856b42f096b9b41dfd0b349af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74075186, jitterRate=0.10380628705024719}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,691 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:29,691 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 073880d856b42f096b9b41dfd0b349af: Running coprocessor pre-open hook at 1732101269621Writing region info on filesystem at 1732101269621Initializing all the Stores at 1732101269625 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269625Cleaning up temporary data from old regions at 1732101269657 (+32 ms)Running coprocessor post-open hooks at 1732101269691 (+34 ms)Region opened successfully at 1732101269691 2024-11-20T11:14:29,693 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af., pid=34, masterSystemTime=1732101269594 2024-11-20T11:14:29,694 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,696 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. 2024-11-20T11:14:29,696 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. 2024-11-20T11:14:29,696 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,696 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:29,696 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a244f5f504c74b0b1dff6963a37aff0, NAME => 'test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-20T11:14:29,697 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,697 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,697 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,697 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,699 INFO [StoreOpener-53a2311878635447fc5975caced0a7d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07feff3ab7eab5622741a0364ece2464, server=448ea6c54a8c,43507,1732101264298 because future has completed 2024-11-20T11:14:29,701 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=073880d856b42f096b9b41dfd0b349af, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 073880d856b42f096b9b41dfd0b349af, server=448ea6c54a8c,35491,1732101264505 because future has completed 2024-11-20T11:14:29,704 INFO [StoreOpener-53a2311878635447fc5975caced0a7d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53a2311878635447fc5975caced0a7d4 columnFamilyName family 2024-11-20T11:14:29,705 DEBUG [StoreOpener-53a2311878635447fc5975caced0a7d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,707 INFO [StoreOpener-53a2311878635447fc5975caced0a7d4-1 {}] regionserver.HStore(327): Store=53a2311878635447fc5975caced0a7d4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,707 INFO [StoreOpener-1a244f5f504c74b0b1dff6963a37aff0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,707 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,707 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ec000da7633d9b0b0b955ae7aa7e962f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,708 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,708 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened ec000da7633d9b0b0b955ae7aa7e962f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65607113, jitterRate=-0.02237783372402191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,709 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:29,709 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for ec000da7633d9b0b0b955ae7aa7e962f: Running coprocessor pre-open hook at 1732101269663Writing region info on filesystem at 1732101269663Initializing all the Stores at 1732101269665 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269665Cleaning up temporary data from old regions at 1732101269691 (+26 ms)Running coprocessor post-open hooks at 1732101269709 (+18 ms)Region opened successfully at 1732101269709 2024-11-20T11:14:29,709 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,710 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,710 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,711 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f., pid=38, masterSystemTime=1732101269616 2024-11-20T11:14:29,711 INFO [StoreOpener-1a244f5f504c74b0b1dff6963a37aff0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a244f5f504c74b0b1dff6963a37aff0 columnFamilyName family 2024-11-20T11:14:29,711 DEBUG [StoreOpener-1a244f5f504c74b0b1dff6963a37aff0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=24 2024-11-20T11:14:29,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 07feff3ab7eab5622741a0364ece2464, server=448ea6c54a8c,43507,1732101264298 in 322 msec 2024-11-20T11:14:29,714 INFO [StoreOpener-1a244f5f504c74b0b1dff6963a37aff0-1 {}] regionserver.HStore(327): Store=1a244f5f504c74b0b1dff6963a37aff0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,714 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,714 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:29,714 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:29,714 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:29,715 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 171d75c7056fb7f3c57a7e0ca3724f6d, NAME => 'test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-20T11:14:29,715 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,715 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,715 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,715 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,715 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,715 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,716 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=07feff3ab7eab5622741a0364ece2464, ASSIGN in 594 msec 2024-11-20T11:14:29,717 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,717 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,719 INFO [StoreOpener-171d75c7056fb7f3c57a7e0ca3724f6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec000da7633d9b0b0b955ae7aa7e962f, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:29,729 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,729 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/53a2311878635447fc5975caced0a7d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,730 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 53a2311878635447fc5975caced0a7d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68164841, jitterRate=0.015735283493995667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,731 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:29,731 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 53a2311878635447fc5975caced0a7d4: Running coprocessor pre-open hook at 1732101269688Writing region info on filesystem at 1732101269688Initializing all the Stores at 1732101269691 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269691Cleaning up temporary data from old regions at 1732101269710 (+19 ms)Running coprocessor post-open hooks at 1732101269731 (+21 ms)Region opened successfully at 1732101269731 2024-11-20T11:14:29,732 INFO [StoreOpener-171d75c7056fb7f3c57a7e0ca3724f6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 171d75c7056fb7f3c57a7e0ca3724f6d columnFamilyName family 2024-11-20T11:14:29,732 DEBUG [StoreOpener-171d75c7056fb7f3c57a7e0ca3724f6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,738 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4., pid=31, masterSystemTime=1732101269528 2024-11-20T11:14:29,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=14 2024-11-20T11:14:29,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 073880d856b42f096b9b41dfd0b349af, server=448ea6c54a8c,35491,1732101264505 in 283 msec 2024-11-20T11:14:29,739 INFO [StoreOpener-171d75c7056fb7f3c57a7e0ca3724f6d-1 {}] regionserver.HStore(327): Store=171d75c7056fb7f3c57a7e0ca3724f6d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,741 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN in 619 msec 2024-11-20T11:14:29,742 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:29,742 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:29,742 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:29,742 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 19c7dd4cb6560dd941c762414bfe9afa, NAME => 'test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-20T11:14:29,743 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,743 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,743 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,743 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,743 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=53a2311878635447fc5975caced0a7d4, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,747 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,748 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53a2311878635447fc5975caced0a7d4, server=448ea6c54a8c,43507,1732101264298 because future has completed 2024-11-20T11:14:29,748 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,749 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1a244f5f504c74b0b1dff6963a37aff0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,750 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 1a244f5f504c74b0b1dff6963a37aff0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69364489, jitterRate=0.03361143171787262}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,750 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:29,750 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 1a244f5f504c74b0b1dff6963a37aff0: Running coprocessor pre-open hook at 1732101269697Writing region info on filesystem at 1732101269697Initializing all the Stores at 1732101269699 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269699Cleaning up temporary data from old regions at 1732101269717 (+18 ms)Running coprocessor post-open hooks at 1732101269750 (+33 ms)Region opened successfully at 1732101269750 2024-11-20T11:14:29,751 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,752 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0., pid=43, masterSystemTime=1732101269594 2024-11-20T11:14:29,755 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:29,755 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:29,756 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1. 2024-11-20T11:14:29,756 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b5da0de9b95a9822f9526f93d4d7af1, NAME => 'test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-20T11:14:29,756 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,756 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,757 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,757 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=19 2024-11-20T11:14:29,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure ec000da7633d9b0b0b955ae7aa7e962f, server=448ea6c54a8c,35099,1732101264437 in 271 msec 2024-11-20T11:14:29,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ec000da7633d9b0b0b955ae7aa7e962f, ASSIGN in 639 msec 2024-11-20T11:14:29,763 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,765 INFO [StoreOpener-19c7dd4cb6560dd941c762414bfe9afa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,35491,1732101264505 because future has completed 2024-11-20T11:14:29,770 INFO [StoreOpener-19c7dd4cb6560dd941c762414bfe9afa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19c7dd4cb6560dd941c762414bfe9afa columnFamilyName family 2024-11-20T11:14:29,770 DEBUG [StoreOpener-19c7dd4cb6560dd941c762414bfe9afa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,771 INFO [StoreOpener-19c7dd4cb6560dd941c762414bfe9afa-1 {}] regionserver.HStore(327): Store=19c7dd4cb6560dd941c762414bfe9afa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,771 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,774 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=448ea6c54a8c,35491,1732101264505, table=test, region=1a244f5f504c74b0b1dff6963a37aff0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T11:14:29,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-20T11:14:29,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 53a2311878635447fc5975caced0a7d4, server=448ea6c54a8c,43507,1732101264298 in 410 msec 2024-11-20T11:14:29,775 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,776 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,776 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/171d75c7056fb7f3c57a7e0ca3724f6d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,776 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,776 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,777 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 171d75c7056fb7f3c57a7e0ca3724f6d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61548537, jitterRate=-0.08285532891750336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,777 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:29,778 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 171d75c7056fb7f3c57a7e0ca3724f6d: Running coprocessor pre-open hook at 1732101269716Writing region info on filesystem at 1732101269716Initializing all the Stores at 1732101269717 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269717Cleaning up temporary data from old regions at 1732101269748 (+31 ms)Running coprocessor post-open hooks at 1732101269777 (+29 ms)Region opened successfully at 1732101269778 (+1 ms) 2024-11-20T11:14:29,780 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d., pid=39, masterSystemTime=1732101269616 2024-11-20T11:14:29,781 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=53a2311878635447fc5975caced0a7d4, ASSIGN in 656 msec 2024-11-20T11:14:29,783 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:29,783 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:29,783 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:29,784 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => fb17647ae131913beb1233c85f78d4dc, NAME => 'test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-20T11:14:29,784 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,784 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,784 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,784 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 171d75c7056fb7f3c57a7e0ca3724f6d, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:29,795 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/19c7dd4cb6560dd941c762414bfe9afa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,796 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 19c7dd4cb6560dd941c762414bfe9afa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65533873, jitterRate=-0.023469194769859314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,796 INFO [StoreOpener-fb17647ae131913beb1233c85f78d4dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,796 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:29,796 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 19c7dd4cb6560dd941c762414bfe9afa: Running coprocessor pre-open hook at 1732101269743Writing region info on filesystem at 1732101269743Initializing all the Stores at 1732101269747 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269747Cleaning up temporary data from old regions at 1732101269777 (+30 ms)Running coprocessor post-open hooks at 1732101269796 (+19 ms)Region opened successfully at 1732101269796 2024-11-20T11:14:29,797 INFO [StoreOpener-2b5da0de9b95a9822f9526f93d4d7af1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,797 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa., pid=41, masterSystemTime=1732101269681 2024-11-20T11:14:29,799 INFO [StoreOpener-fb17647ae131913beb1233c85f78d4dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb17647ae131913beb1233c85f78d4dc columnFamilyName family 2024-11-20T11:14:29,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=21 2024-11-20T11:14:29,799 DEBUG [StoreOpener-fb17647ae131913beb1233c85f78d4dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,799 INFO [StoreOpener-2b5da0de9b95a9822f9526f93d4d7af1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b5da0de9b95a9822f9526f93d4d7af1 columnFamilyName family 2024-11-20T11:14:29,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 171d75c7056fb7f3c57a7e0ca3724f6d, server=448ea6c54a8c,35099,1732101264437 in 304 msec 2024-11-20T11:14:29,800 DEBUG [StoreOpener-2b5da0de9b95a9822f9526f93d4d7af1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,800 INFO [StoreOpener-fb17647ae131913beb1233c85f78d4dc-1 {}] regionserver.HStore(327): Store=fb17647ae131913beb1233c85f78d4dc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,801 INFO [StoreOpener-2b5da0de9b95a9822f9526f93d4d7af1-1 {}] regionserver.HStore(327): Store=2b5da0de9b95a9822f9526f93d4d7af1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,801 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,801 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:29,801 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,801 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:29,801 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:29,801 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e8a8778f20ffb0d01039b06a807ee6a, NAME => 'test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-20T11:14:29,802 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,802 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,802 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,802 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,802 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,803 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,803 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,804 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,804 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,804 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=171d75c7056fb7f3c57a7e0ca3724f6d, ASSIGN in 681 msec 2024-11-20T11:14:29,805 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,805 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,806 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,807 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,808 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,818 INFO [StoreOpener-4e8a8778f20ffb0d01039b06a807ee6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19c7dd4cb6560dd941c762414bfe9afa, server=448ea6c54a8c,43507,1732101264298 because future has completed 2024-11-20T11:14:29,822 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/2b5da0de9b95a9822f9526f93d4d7af1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=10 2024-11-20T11:14:29,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,35491,1732101264505 in 236 msec 2024-11-20T11:14:29,823 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 2b5da0de9b95a9822f9526f93d4d7af1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74830534, jitterRate=0.11506184935569763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,823 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b5da0de9b95a9822f9526f93d4d7af1 2024-11-20T11:14:29,823 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 2b5da0de9b95a9822f9526f93d4d7af1: Running coprocessor pre-open hook at 1732101269757Writing region info on filesystem at 1732101269757Initializing all the Stores at 1732101269760 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269760Cleaning up temporary data from old regions at 1732101269804 (+44 ms)Running coprocessor post-open hooks at 1732101269823 (+19 ms)Region opened successfully at 1732101269823 2024-11-20T11:14:29,824 INFO [StoreOpener-4e8a8778f20ffb0d01039b06a807ee6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e8a8778f20ffb0d01039b06a807ee6a columnFamilyName family 2024-11-20T11:14:29,824 DEBUG [StoreOpener-4e8a8778f20ffb0d01039b06a807ee6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,825 INFO [StoreOpener-4e8a8778f20ffb0d01039b06a807ee6a-1 {}] regionserver.HStore(327): Store=4e8a8778f20ffb0d01039b06a807ee6a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,825 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/fb17647ae131913beb1233c85f78d4dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,826 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened fb17647ae131913beb1233c85f78d4dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68258469, jitterRate=0.017130449414253235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,826 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,827 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:29,827 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1., pid=33, masterSystemTime=1732101269594 2024-11-20T11:14:29,827 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for fb17647ae131913beb1233c85f78d4dc: Running coprocessor pre-open hook at 1732101269784Writing region info on filesystem at 1732101269784Initializing all the Stores at 1732101269786 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269786Cleaning up temporary data from old regions at 1732101269805 (+19 ms)Running coprocessor post-open hooks at 1732101269827 (+22 ms)Region opened successfully at 1732101269827 2024-11-20T11:14:29,832 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,833 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,833 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc., pid=44, masterSystemTime=1732101269616 2024-11-20T11:14:29,833 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,833 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,834 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1. 2024-11-20T11:14:29,834 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,vvv,1732101267801.2b5da0de9b95a9822f9526f93d4d7af1. 2024-11-20T11:14:29,834 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:29,834 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => d221d096301219259fde4786f6bb7402, NAME => 'test,lll,1732101267801.d221d096301219259fde4786f6bb7402.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-20T11:14:29,835 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,835 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,lll,1732101267801.d221d096301219259fde4786f6bb7402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,835 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,835 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,837 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:29,837 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:29,837 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:29,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN in 703 msec 2024-11-20T11:14:29,838 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => ddf1f8c8ecace91d120dbb68cbf18fb1, NAME => 'test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-20T11:14:29,838 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,838 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,838 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,838 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,838 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,839 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,841 INFO [StoreOpener-d221d096301219259fde4786f6bb7402-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb17647ae131913beb1233c85f78d4dc, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:29,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=30 2024-11-20T11:14:29,853 INFO [StoreOpener-d221d096301219259fde4786f6bb7402-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d221d096301219259fde4786f6bb7402 columnFamilyName family 2024-11-20T11:14:29,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure fb17647ae131913beb1233c85f78d4dc, server=448ea6c54a8c,35099,1732101264437 in 254 msec 2024-11-20T11:14:29,853 DEBUG [StoreOpener-d221d096301219259fde4786f6bb7402-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,854 INFO [StoreOpener-d221d096301219259fde4786f6bb7402-1 {}] regionserver.HStore(327): Store=d221d096301219259fde4786f6bb7402/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,854 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,849 INFO [StoreOpener-ddf1f8c8ecace91d120dbb68cbf18fb1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,856 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,856 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fb17647ae131913beb1233c85f78d4dc, ASSIGN in 734 msec 2024-11-20T11:14:29,857 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,857 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,859 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/4e8a8778f20ffb0d01039b06a807ee6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,860 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,860 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 4e8a8778f20ffb0d01039b06a807ee6a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69020439, jitterRate=0.028484687209129333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,861 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:29,861 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 4e8a8778f20ffb0d01039b06a807ee6a: Running coprocessor pre-open hook at 1732101269802Writing region info on filesystem at 1732101269802Initializing all the Stores at 1732101269804 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269804Cleaning up temporary data from old regions at 1732101269833 (+29 ms)Running coprocessor post-open hooks at 1732101269861 (+28 ms)Region opened successfully at 1732101269861 2024-11-20T11:14:29,861 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,861 INFO [StoreOpener-ddf1f8c8ecace91d120dbb68cbf18fb1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddf1f8c8ecace91d120dbb68cbf18fb1 columnFamilyName family 2024-11-20T11:14:29,861 DEBUG [StoreOpener-ddf1f8c8ecace91d120dbb68cbf18fb1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,862 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a., pid=49, masterSystemTime=1732101269681 2024-11-20T11:14:29,862 INFO [StoreOpener-ddf1f8c8ecace91d120dbb68cbf18fb1-1 {}] regionserver.HStore(327): Store=ddf1f8c8ecace91d120dbb68cbf18fb1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,862 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35491,1732101264505 because future has completed 2024-11-20T11:14:29,865 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,866 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,867 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:29,867 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:29,867 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:29,867 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 94ab7bd7c2c2041d4db8eb8309fa9cea, NAME => 'test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-20T11:14:29,868 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,868 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,868 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,868 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,869 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,869 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,869 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/d221d096301219259fde4786f6bb7402/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,871 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened d221d096301219259fde4786f6bb7402; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68433235, jitterRate=0.019734665751457214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,871 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:29,871 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for d221d096301219259fde4786f6bb7402: Running coprocessor pre-open hook at 1732101269835Writing region info on filesystem at 1732101269835Initializing all the Stores at 1732101269837 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269837Cleaning up temporary data from old regions at 1732101269857 (+20 ms)Running coprocessor post-open hooks at 1732101269871 (+14 ms)Region opened successfully at 1732101269871 2024-11-20T11:14:29,873 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732101267801.d221d096301219259fde4786f6bb7402., pid=45, masterSystemTime=1732101269763 2024-11-20T11:14:29,873 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,874 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,878 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=448ea6c54a8c,35491,1732101264505, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T11:14:29,879 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:29,879 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:29,879 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:29,879 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 400f5d1df45ccf5378a59a8cd78bb9d8, NAME => 'test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-20T11:14:29,880 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,880 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,880 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,880 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,881 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=test, region=19c7dd4cb6560dd941c762414bfe9afa. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T11:14:29,881 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=test, region=4e8a8778f20ffb0d01039b06a807ee6a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T11:14:29,882 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=49, ppid=13, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4e8a8778f20ffb0d01039b06a807ee6a, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=13 2024-11-20T11:14:29,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e8a8778f20ffb0d01039b06a807ee6a, server=448ea6c54a8c,43507,1732101264298 in 220 msec 2024-11-20T11:14:29,891 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ddf1f8c8ecace91d120dbb68cbf18fb1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4e8a8778f20ffb0d01039b06a807ee6a, ASSIGN in 769 msec 2024-11-20T11:14:29,891 INFO [StoreOpener-94ab7bd7c2c2041d4db8eb8309fa9cea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,892 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened ddf1f8c8ecace91d120dbb68cbf18fb1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67196020, jitterRate=0.0012987256050109863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,892 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:29,892 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for ddf1f8c8ecace91d120dbb68cbf18fb1: Running coprocessor pre-open hook at 1732101269838Writing region info on filesystem at 1732101269838Initializing all the Stores at 1732101269840 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269840Cleaning up temporary data from old regions at 1732101269869 (+29 ms)Running coprocessor post-open hooks at 1732101269892 (+23 ms)Region opened successfully at 1732101269892 2024-11-20T11:14:29,893 INFO [StoreOpener-400f5d1df45ccf5378a59a8cd78bb9d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,894 INFO [StoreOpener-94ab7bd7c2c2041d4db8eb8309fa9cea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94ab7bd7c2c2041d4db8eb8309fa9cea columnFamilyName family 2024-11-20T11:14:29,894 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1., pid=36, masterSystemTime=1732101269616 2024-11-20T11:14:29,894 DEBUG [StoreOpener-94ab7bd7c2c2041d4db8eb8309fa9cea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,894 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=d221d096301219259fde4786f6bb7402, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,895 INFO [StoreOpener-400f5d1df45ccf5378a59a8cd78bb9d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 400f5d1df45ccf5378a59a8cd78bb9d8 columnFamilyName family 2024-11-20T11:14:29,895 DEBUG [StoreOpener-400f5d1df45ccf5378a59a8cd78bb9d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,896 INFO [StoreOpener-400f5d1df45ccf5378a59a8cd78bb9d8-1 {}] regionserver.HStore(327): Store=400f5d1df45ccf5378a59a8cd78bb9d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,896 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,897 INFO [StoreOpener-94ab7bd7c2c2041d4db8eb8309fa9cea-1 {}] regionserver.HStore(327): Store=94ab7bd7c2c2041d4db8eb8309fa9cea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,897 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,898 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,898 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,898 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:29,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,35491,1732101264505 because future has completed 2024-11-20T11:14:29,898 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,898 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:29,899 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:29,899 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => bac309a4ff803986756ba76688d03707, NAME => 'test,eee,1732101267801.bac309a4ff803986756ba76688d03707.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-20T11:14:29,899 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,899 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,899 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,899 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,899 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,eee,1732101267801.bac309a4ff803986756ba76688d03707.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,900 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,900 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,900 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,900 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,906 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,907 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=16 2024-11-20T11:14:29,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,35491,1732101264505 in 293 msec 2024-11-20T11:14:29,913 INFO [StoreOpener-bac309a4ff803986756ba76688d03707-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN in 792 msec 2024-11-20T11:14:29,918 INFO [StoreOpener-bac309a4ff803986756ba76688d03707-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bac309a4ff803986756ba76688d03707 columnFamilyName family 2024-11-20T11:14:29,918 DEBUG [StoreOpener-bac309a4ff803986756ba76688d03707-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,919 INFO [StoreOpener-bac309a4ff803986756ba76688d03707-1 {}] regionserver.HStore(327): Store=bac309a4ff803986756ba76688d03707/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,919 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,920 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,921 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,922 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,922 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddf1f8c8ecace91d120dbb68cbf18fb1, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:29,928 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/94ab7bd7c2c2041d4db8eb8309fa9cea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,928 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/400f5d1df45ccf5378a59a8cd78bb9d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,930 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 94ab7bd7c2c2041d4db8eb8309fa9cea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74463612, jitterRate=0.10959428548812866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,930 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:29,930 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,930 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 94ab7bd7c2c2041d4db8eb8309fa9cea: Running coprocessor pre-open hook at 1732101269868Writing region info on filesystem at 1732101269868Initializing all the Stores at 1732101269874 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269874Cleaning up temporary data from old regions at 1732101269900 (+26 ms)Running coprocessor post-open hooks at 1732101269930 (+30 ms)Region opened successfully at 1732101269930 2024-11-20T11:14:29,931 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 400f5d1df45ccf5378a59a8cd78bb9d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64350856, jitterRate=-0.04109752178192139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,931 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:29,931 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 400f5d1df45ccf5378a59a8cd78bb9d8: Running coprocessor pre-open hook at 1732101269880Writing region info on filesystem at 1732101269880Initializing all the Stores at 1732101269882 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269882Cleaning up temporary data from old regions at 1732101269899 (+17 ms)Running coprocessor post-open hooks at 1732101269931 (+32 ms)Region opened successfully at 1732101269931 2024-11-20T11:14:29,932 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea., pid=50, masterSystemTime=1732101269681 2024-11-20T11:14:29,933 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8., pid=48, masterSystemTime=1732101269763 2024-11-20T11:14:29,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=26 2024-11-20T11:14:29,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35491,1732101264505 in 495 msec 2024-11-20T11:14:29,937 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:29,937 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:29,937 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:29,937 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 10e6b7dac8e72b3715b09f737ea5dce3, NAME => 'test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-20T11:14:29,938 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,938 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,938 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,938 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,939 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:29,939 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:29,939 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:29,939 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 6864708d24f9dd54d37dd8e832ca6cb9, NAME => 'test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-20T11:14:29,940 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,940 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,940 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,940 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,943 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/bac309a4ff803986756ba76688d03707/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN in 817 msec 2024-11-20T11:14:29,946 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened bac309a4ff803986756ba76688d03707; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68301712, jitterRate=0.01777482032775879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,946 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bac309a4ff803986756ba76688d03707 2024-11-20T11:14:29,946 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for bac309a4ff803986756ba76688d03707: Running coprocessor pre-open hook at 1732101269900Writing region info on filesystem at 1732101269900Initializing all the Stores at 1732101269904 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269904Cleaning up temporary data from old regions at 1732101269922 (+18 ms)Running coprocessor post-open hooks at 1732101269946 (+24 ms)Region opened successfully at 1732101269946 2024-11-20T11:14:29,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:29,948 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732101267801.bac309a4ff803986756ba76688d03707., pid=42, masterSystemTime=1732101269616 2024-11-20T11:14:29,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94ab7bd7c2c2041d4db8eb8309fa9cea, server=448ea6c54a8c,43507,1732101264298 because future has completed 2024-11-20T11:14:29,953 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:29,953 INFO [StoreOpener-10e6b7dac8e72b3715b09f737ea5dce3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,953 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:29,953 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:29,953 INFO [StoreOpener-6864708d24f9dd54d37dd8e832ca6cb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,953 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 16717e829fb199e0c26298a5116a0b4e, NAME => 'test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-20T11:14:29,954 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,954 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:29,954 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,954 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,955 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:29,956 INFO [StoreOpener-10e6b7dac8e72b3715b09f737ea5dce3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10e6b7dac8e72b3715b09f737ea5dce3 columnFamilyName family 2024-11-20T11:14:29,956 DEBUG [StoreOpener-10e6b7dac8e72b3715b09f737ea5dce3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,958 INFO [StoreOpener-10e6b7dac8e72b3715b09f737ea5dce3-1 {}] regionserver.HStore(327): Store=10e6b7dac8e72b3715b09f737ea5dce3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,958 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,961 INFO [StoreOpener-6864708d24f9dd54d37dd8e832ca6cb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6864708d24f9dd54d37dd8e832ca6cb9 columnFamilyName family 2024-11-20T11:14:29,961 DEBUG [StoreOpener-6864708d24f9dd54d37dd8e832ca6cb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,961 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,963 INFO [StoreOpener-6864708d24f9dd54d37dd8e832ca6cb9-1 {}] regionserver.HStore(327): Store=6864708d24f9dd54d37dd8e832ca6cb9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,963 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,963 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,964 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,964 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,964 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,965 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,966 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,966 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,35491,1732101264505 because future has completed 2024-11-20T11:14:29,967 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=bac309a4ff803986756ba76688d03707, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:29,969 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:29,970 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure bac309a4ff803986756ba76688d03707, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:29,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=28 2024-11-20T11:14:29,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,35491,1732101264505 in 331 msec 2024-11-20T11:14:29,979 INFO [StoreOpener-16717e829fb199e0c26298a5116a0b4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN in 857 msec 2024-11-20T11:14:29,987 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=448ea6c54a8c,35099,1732101264437, table=test, region=ddf1f8c8ecace91d120dbb68cbf18fb1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T11:14:29,988 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=test, region=19c7dd4cb6560dd941c762414bfe9afa. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T11:14:29,988 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=test, region=94ab7bd7c2c2041d4db8eb8309fa9cea. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T11:14:29,989 INFO [StoreOpener-16717e829fb199e0c26298a5116a0b4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16717e829fb199e0c26298a5116a0b4e columnFamilyName family 2024-11-20T11:14:29,989 DEBUG [StoreOpener-16717e829fb199e0c26298a5116a0b4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:29,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=9 2024-11-20T11:14:29,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure bac309a4ff803986756ba76688d03707, server=448ea6c54a8c,35099,1732101264437 in 423 msec 2024-11-20T11:14:29,991 INFO [StoreOpener-16717e829fb199e0c26298a5116a0b4e-1 {}] regionserver.HStore(327): Store=16717e829fb199e0c26298a5116a0b4e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:29,992 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,993 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,993 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/6864708d24f9dd54d37dd8e832ca6cb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:29,994 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bac309a4ff803986756ba76688d03707, ASSIGN in 872 msec 2024-11-20T11:14:29,994 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 6864708d24f9dd54d37dd8e832ca6cb9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59442643, jitterRate=-0.11423559486865997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:29,994 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,994 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:29,994 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:29,995 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 6864708d24f9dd54d37dd8e832ca6cb9: Running coprocessor pre-open hook at 1732101269940Writing region info on filesystem at 1732101269940Initializing all the Stores at 1732101269945 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269945Cleaning up temporary data from old regions at 1732101269966 (+21 ms)Running coprocessor post-open hooks at 1732101269994 (+28 ms)Region opened successfully at 1732101269994 2024-11-20T11:14:29,997 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:30,000 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9., pid=51, masterSystemTime=1732101269836 2024-11-20T11:14:30,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=17 2024-11-20T11:14:30,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 94ab7bd7c2c2041d4db8eb8309fa9cea, server=448ea6c54a8c,43507,1732101264298 in 324 msec 2024-11-20T11:14:30,003 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:30,003 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:30,007 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:30,010 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/10e6b7dac8e72b3715b09f737ea5dce3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,010 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/16717e829fb199e0c26298a5116a0b4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,011 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 10e6b7dac8e72b3715b09f737ea5dce3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65066785, jitterRate=-0.030429348349571228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,011 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 16717e829fb199e0c26298a5116a0b4e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66038456, jitterRate=-0.015950322151184082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,011 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:30,011 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:30,011 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 10e6b7dac8e72b3715b09f737ea5dce3: Running coprocessor pre-open hook at 1732101269938Writing region info on filesystem at 1732101269938Initializing all the Stores at 1732101269942 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269942Cleaning up temporary data from old regions at 1732101269964 (+22 ms)Running coprocessor post-open hooks at 1732101270011 (+47 ms)Region opened successfully at 1732101270011 2024-11-20T11:14:30,011 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 16717e829fb199e0c26298a5116a0b4e: Running coprocessor pre-open hook at 1732101269954Writing region info on filesystem at 1732101269954Initializing all the Stores at 1732101269962 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101269962Cleaning up temporary data from old regions at 1732101269994 (+32 ms)Running coprocessor post-open hooks at 1732101270011 (+17 ms)Region opened successfully at 1732101270011 2024-11-20T11:14:30,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6864708d24f9dd54d37dd8e832ca6cb9, server=448ea6c54a8c,43507,1732101264298 because future has completed 2024-11-20T11:14:30,012 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3., pid=47, masterSystemTime=1732101269763 2024-11-20T11:14:30,018 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e., pid=40, masterSystemTime=1732101269616 2024-11-20T11:14:30,019 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:30,019 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:30,019 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6. 2024-11-20T11:14:30,020 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => c8b0425cde259974fdc42bd0afa1e9d6, NAME => 'test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-20T11:14:30,020 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,020 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:30,020 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,020 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=94ab7bd7c2c2041d4db8eb8309fa9cea, ASSIGN in 893 msec 2024-11-20T11:14:30,023 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:30,023 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:30,023 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:30,024 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 031ddda36eb9bb40423f7c9f4322100c, NAME => 'test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-20T11:14:30,024 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,024 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:30,024 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,024 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:30,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,35491,1732101264505 because future has completed 2024-11-20T11:14:30,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:30,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16717e829fb199e0c26298a5116a0b4e, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:30,044 INFO [StoreOpener-c8b0425cde259974fdc42bd0afa1e9d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,045 INFO [StoreOpener-031ddda36eb9bb40423f7c9f4322100c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,048 INFO [StoreOpener-c8b0425cde259974fdc42bd0afa1e9d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8b0425cde259974fdc42bd0afa1e9d6 columnFamilyName family 2024-11-20T11:14:30,048 DEBUG [StoreOpener-c8b0425cde259974fdc42bd0afa1e9d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:30,049 INFO [StoreOpener-c8b0425cde259974fdc42bd0afa1e9d6-1 {}] regionserver.HStore(327): Store=c8b0425cde259974fdc42bd0afa1e9d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:30,049 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,050 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=25 2024-11-20T11:14:30,051 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T11:14:30,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 16717e829fb199e0c26298a5116a0b4e, server=448ea6c54a8c,35099,1732101264437 in 536 msec 2024-11-20T11:14:30,052 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,052 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,055 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=16717e829fb199e0c26298a5116a0b4e, ASSIGN in 933 msec 2024-11-20T11:14:30,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=23 2024-11-20T11:14:30,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,35491,1732101264505 in 432 msec 2024-11-20T11:14:30,065 INFO [StoreOpener-031ddda36eb9bb40423f7c9f4322100c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 031ddda36eb9bb40423f7c9f4322100c columnFamilyName family 2024-11-20T11:14:30,065 DEBUG [StoreOpener-031ddda36eb9bb40423f7c9f4322100c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:30,067 INFO [StoreOpener-031ddda36eb9bb40423f7c9f4322100c-1 {}] regionserver.HStore(327): Store=031ddda36eb9bb40423f7c9f4322100c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:30,067 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,068 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN in 946 msec 2024-11-20T11:14:30,069 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,070 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,070 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,072 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/c8b0425cde259974fdc42bd0afa1e9d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,072 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened c8b0425cde259974fdc42bd0afa1e9d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61232918, jitterRate=-0.08755841851234436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,073 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8b0425cde259974fdc42bd0afa1e9d6 2024-11-20T11:14:30,073 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for c8b0425cde259974fdc42bd0afa1e9d6: Running coprocessor pre-open hook at 1732101270021Writing region info on filesystem at 1732101270021Initializing all the Stores at 1732101270022 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101270023 (+1 ms)Cleaning up temporary data from old regions at 1732101270052 (+29 ms)Running coprocessor post-open hooks at 1732101270073 (+21 ms)Region opened successfully at 1732101270073 2024-11-20T11:14:30,074 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,074 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6., pid=46, masterSystemTime=1732101269763 2024-11-20T11:14:30,079 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6. 2024-11-20T11:14:30,079 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,nnn,1732101267801.c8b0425cde259974fdc42bd0afa1e9d6. 2024-11-20T11:14:30,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=22 2024-11-20T11:14:30,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 6864708d24f9dd54d37dd8e832ca6cb9, server=448ea6c54a8c,43507,1732101264298 in 392 msec 2024-11-20T11:14:30,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6864708d24f9dd54d37dd8e832ca6cb9, ASSIGN in 961 msec 2024-11-20T11:14:30,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:30,093 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/031ddda36eb9bb40423f7c9f4322100c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,094 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 031ddda36eb9bb40423f7c9f4322100c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66319170, jitterRate=-0.011767357587814331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,094 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:30,094 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 031ddda36eb9bb40423f7c9f4322100c: Running coprocessor pre-open hook at 1732101270024Writing region info on filesystem at 1732101270024Initializing all the Stores at 1732101270031 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101270031Cleaning up temporary data from old regions at 1732101270070 (+39 ms)Running coprocessor post-open hooks at 1732101270094 (+24 ms)Region opened successfully at 1732101270094 2024-11-20T11:14:30,095 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c., pid=35, masterSystemTime=1732101269616 2024-11-20T11:14:30,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35491,1732101264505 because future has completed 2024-11-20T11:14:30,099 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:30,100 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:30,100 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:30,100 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 70c2f19bb4361dae17cea153d84e898a, NAME => 'test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-20T11:14:30,100 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,101 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:30,101 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,101 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,104 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=test, region=19c7dd4cb6560dd941c762414bfe9afa. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T11:14:30,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=12 2024-11-20T11:14:30,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure ddf1f8c8ecace91d120dbb68cbf18fb1, server=448ea6c54a8c,35099,1732101264437 in 629 msec 2024-11-20T11:14:30,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ddf1f8c8ecace91d120dbb68cbf18fb1, ASSIGN in 986 msec 2024-11-20T11:14:30,112 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:30,115 INFO [StoreOpener-70c2f19bb4361dae17cea153d84e898a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 031ddda36eb9bb40423f7c9f4322100c, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:30,118 INFO [StoreOpener-70c2f19bb4361dae17cea153d84e898a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70c2f19bb4361dae17cea153d84e898a columnFamilyName family 2024-11-20T11:14:30,118 DEBUG [StoreOpener-70c2f19bb4361dae17cea153d84e898a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:30,119 INFO [StoreOpener-70c2f19bb4361dae17cea153d84e898a-1 {}] regionserver.HStore(327): Store=70c2f19bb4361dae17cea153d84e898a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:30,119 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,121 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,121 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,122 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,122 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=18 2024-11-20T11:14:30,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35491,1732101264505 in 500 msec 2024-11-20T11:14:30,127 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN in 1.0040 sec 2024-11-20T11:14:30,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=20 2024-11-20T11:14:30,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 031ddda36eb9bb40423f7c9f4322100c, server=448ea6c54a8c,35099,1732101264437 in 675 msec 2024-11-20T11:14:30,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=031ddda36eb9bb40423f7c9f4322100c, ASSIGN in 1.0230 sec 2024-11-20T11:14:30,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=11 2024-11-20T11:14:30,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 19c7dd4cb6560dd941c762414bfe9afa, server=448ea6c54a8c,43507,1732101264298 in 636 msec 2024-11-20T11:14:30,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=19c7dd4cb6560dd941c762414bfe9afa, ASSIGN in 1.0450 sec 2024-11-20T11:14:30,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 349e25c47fdd7d8457f465670572f372, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:30,171 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/70c2f19bb4361dae17cea153d84e898a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,172 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 70c2f19bb4361dae17cea153d84e898a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69344453, jitterRate=0.03331287205219269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,172 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:30,172 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 70c2f19bb4361dae17cea153d84e898a: Running coprocessor pre-open hook at 1732101270101Writing region info on filesystem at 1732101270102 (+1 ms)Initializing all the Stores at 1732101270103 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101270103Cleaning up temporary data from old regions at 1732101270122 (+19 ms)Running coprocessor post-open hooks at 1732101270172 (+50 ms)Region opened successfully at 1732101270172 2024-11-20T11:14:30,174 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a., pid=37, masterSystemTime=1732101269616 2024-11-20T11:14:30,177 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cb4960297e33c5789bed6704818bf798, ASSIGN 2024-11-20T11:14:30,177 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:30,177 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:30,178 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cb4960297e33c5789bed6704818bf798, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:30,181 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:30,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70c2f19bb4361dae17cea153d84e898a, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:30,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=15 2024-11-20T11:14:30,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 70c2f19bb4361dae17cea153d84e898a, server=448ea6c54a8c,35099,1732101264437 in 715 msec 2024-11-20T11:14:30,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=70c2f19bb4361dae17cea153d84e898a, ASSIGN in 1.0730 sec 2024-11-20T11:14:30,195 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:30,196 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35491,1732101264505; forceNewPlan=false, retain=false 2024-11-20T11:14:30,197 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab39ce583e7145069013a6bec41b40c0, ASSIGN 2024-11-20T11:14:30,204 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ab39ce583e7145069013a6bec41b40c0, ASSIGN; state=OFFLINE, location=448ea6c54a8c,43507,1732101264298; forceNewPlan=false, retain=false 2024-11-20T11:14:30,208 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e9ad74913c92386ef2a8194575ccf6d, ASSIGN 2024-11-20T11:14:30,210 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1e9ad74913c92386ef2a8194575ccf6d, ASSIGN; state=OFFLINE, location=448ea6c54a8c,35099,1732101264437; forceNewPlan=false, retain=false 2024-11-20T11:14:30,329 INFO [448ea6c54a8c:38459 {}] balancer.BaseLoadBalancer(388): Reassigned 4 regions. 4 retained the pre-restart assignment. 2024-11-20T11:14:30,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=cb4960297e33c5789bed6704818bf798, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:30,331 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=ab39ce583e7145069013a6bec41b40c0, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:30,332 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:30,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:30,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cb4960297e33c5789bed6704818bf798, ASSIGN because future has completed 2024-11-20T11:14:30,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb4960297e33c5789bed6704818bf798, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:30,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ab39ce583e7145069013a6bec41b40c0, ASSIGN because future has completed 2024-11-20T11:14:30,342 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:30,343 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 349e25c47fdd7d8457f465670572f372, NAME => 'test,www,1732101267801.349e25c47fdd7d8457f465670572f372.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-20T11:14:30,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN because future has completed 2024-11-20T11:14:30,343 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,343 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,www,1732101267801.349e25c47fdd7d8457f465670572f372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:30,343 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,344 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1e9ad74913c92386ef2a8194575ccf6d, ASSIGN because future has completed 2024-11-20T11:14:30,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e9ad74913c92386ef2a8194575ccf6d, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:30,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35491,1732101264505}] 2024-11-20T11:14:30,357 INFO [StoreOpener-349e25c47fdd7d8457f465670572f372-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab39ce583e7145069013a6bec41b40c0, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:30,372 INFO [StoreOpener-349e25c47fdd7d8457f465670572f372-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 349e25c47fdd7d8457f465670572f372 columnFamilyName family 2024-11-20T11:14:30,372 DEBUG [StoreOpener-349e25c47fdd7d8457f465670572f372-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:30,373 INFO [StoreOpener-349e25c47fdd7d8457f465670572f372-1 {}] regionserver.HStore(327): Store=349e25c47fdd7d8457f465670572f372/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:30,373 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,374 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,375 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,376 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,376 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,378 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,409 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/349e25c47fdd7d8457f465670572f372/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,411 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 349e25c47fdd7d8457f465670572f372; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60137775, jitterRate=-0.10387732088565826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,411 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:30,411 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 349e25c47fdd7d8457f465670572f372: Running coprocessor pre-open hook at 1732101270344Writing region info on filesystem at 1732101270344Initializing all the Stores at 1732101270349 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101270349Cleaning up temporary data from old regions at 1732101270376 (+27 ms)Running coprocessor post-open hooks at 1732101270411 (+35 ms)Region opened successfully at 1732101270411 2024-11-20T11:14:30,413 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732101267801.349e25c47fdd7d8457f465670572f372., pid=52, masterSystemTime=1732101270323 2024-11-20T11:14:30,423 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:30,423 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:30,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=349e25c47fdd7d8457f465670572f372, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:30,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 349e25c47fdd7d8457f465670572f372, server=448ea6c54a8c,43507,1732101264298 because future has completed 2024-11-20T11:14:30,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=27 2024-11-20T11:14:30,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 349e25c47fdd7d8457f465670572f372, server=448ea6c54a8c,43507,1732101264298 in 265 msec 2024-11-20T11:14:30,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=349e25c47fdd7d8457f465670572f372, ASSIGN in 1.3180 sec 2024-11-20T11:14:30,502 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:30,502 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => cb4960297e33c5789bed6704818bf798, NAME => 'test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-20T11:14:30,503 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,503 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:30,503 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,503 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,516 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:30,516 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e9ad74913c92386ef2a8194575ccf6d, NAME => 'test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-20T11:14:30,516 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,516 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:30,517 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,517 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,517 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877. 2024-11-20T11:14:30,518 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c331e73b55d2ae38913c70b5317c877, NAME => 'test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-20T11:14:30,518 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,518 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:30,518 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,519 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,520 INFO [StoreOpener-cb4960297e33c5789bed6704818bf798-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,523 INFO [StoreOpener-cb4960297e33c5789bed6704818bf798-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb4960297e33c5789bed6704818bf798 columnFamilyName family 2024-11-20T11:14:30,523 DEBUG [StoreOpener-cb4960297e33c5789bed6704818bf798-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:30,524 INFO [StoreOpener-cb4960297e33c5789bed6704818bf798-1 {}] regionserver.HStore(327): Store=cb4960297e33c5789bed6704818bf798/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:30,524 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,526 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,527 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,528 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,528 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,530 INFO [StoreOpener-1e9ad74913c92386ef2a8194575ccf6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,531 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,532 INFO [StoreOpener-1e9ad74913c92386ef2a8194575ccf6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e9ad74913c92386ef2a8194575ccf6d columnFamilyName family 2024-11-20T11:14:30,532 DEBUG [StoreOpener-1e9ad74913c92386ef2a8194575ccf6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:30,533 INFO [StoreOpener-1e9ad74913c92386ef2a8194575ccf6d-1 {}] regionserver.HStore(327): Store=1e9ad74913c92386ef2a8194575ccf6d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:30,533 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,534 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,535 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,535 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,535 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,538 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,545 INFO [StoreOpener-0c331e73b55d2ae38913c70b5317c877-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,548 INFO [StoreOpener-0c331e73b55d2ae38913c70b5317c877-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c331e73b55d2ae38913c70b5317c877 columnFamilyName family 2024-11-20T11:14:30,549 DEBUG [StoreOpener-0c331e73b55d2ae38913c70b5317c877-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:30,549 INFO [StoreOpener-0c331e73b55d2ae38913c70b5317c877-1 {}] regionserver.HStore(327): Store=0c331e73b55d2ae38913c70b5317c877/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:30,550 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,551 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,551 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,552 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,552 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,554 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,561 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/cb4960297e33c5789bed6704818bf798/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,561 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened cb4960297e33c5789bed6704818bf798; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71614360, jitterRate=0.06713712215423584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,562 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:30,562 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for cb4960297e33c5789bed6704818bf798: Running coprocessor pre-open hook at 1732101270503Writing region info on filesystem at 1732101270503Initializing all the Stores at 1732101270505 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101270505Cleaning up temporary data from old regions at 1732101270528 (+23 ms)Running coprocessor post-open hooks at 1732101270562 (+34 ms)Region opened successfully at 1732101270562 2024-11-20T11:14:30,563 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798., pid=53, masterSystemTime=1732101270496 2024-11-20T11:14:30,567 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:30,567 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:30,568 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:30,568 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => ab39ce583e7145069013a6bec41b40c0, NAME => 'test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-20T11:14:30,568 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,568 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:30,568 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,568 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,568 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=cb4960297e33c5789bed6704818bf798, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:30,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb4960297e33c5789bed6704818bf798, server=448ea6c54a8c,43507,1732101264298 because future has completed 2024-11-20T11:14:30,576 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1e9ad74913c92386ef2a8194575ccf6d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,576 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 1e9ad74913c92386ef2a8194575ccf6d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62204254, jitterRate=-0.07308438420295715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,576 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:30,577 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 1e9ad74913c92386ef2a8194575ccf6d: Running coprocessor pre-open hook at 1732101270517Writing region info on filesystem at 1732101270517Initializing all the Stores at 1732101270520 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101270520Cleaning up temporary data from old regions at 1732101270535 (+15 ms)Running coprocessor post-open hooks at 1732101270576 (+41 ms)Region opened successfully at 1732101270577 (+1 ms) 2024-11-20T11:14:30,579 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d., pid=54, masterSystemTime=1732101270507 2024-11-20T11:14:30,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=8 2024-11-20T11:14:30,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure cb4960297e33c5789bed6704818bf798, server=448ea6c54a8c,43507,1732101264298 in 239 msec 2024-11-20T11:14:30,584 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:30,584 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:30,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cb4960297e33c5789bed6704818bf798, ASSIGN in 1.4610 sec 2024-11-20T11:14:30,586 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/0c331e73b55d2ae38913c70b5317c877/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:30,586 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 0c331e73b55d2ae38913c70b5317c877; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74289565, jitterRate=0.10700078308582306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,586 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0c331e73b55d2ae38913c70b5317c877 2024-11-20T11:14:30,587 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 0c331e73b55d2ae38913c70b5317c877: Running coprocessor pre-open hook at 1732101270519Writing region info on filesystem at 1732101270519Initializing all the Stores at 1732101270520 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101270520Cleaning up temporary data from old regions at 1732101270552 (+32 ms)Running coprocessor post-open hooks at 1732101270586 (+34 ms)Region opened successfully at 1732101270587 (+1 ms) 2024-11-20T11:14:30,589 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877., pid=55, masterSystemTime=1732101270512 2024-11-20T11:14:30,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e9ad74913c92386ef2a8194575ccf6d, server=448ea6c54a8c,35099,1732101264437 because future has completed 2024-11-20T11:14:30,592 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877. 2024-11-20T11:14:30,592 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877. 2024-11-20T11:14:30,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=5 2024-11-20T11:14:30,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e9ad74913c92386ef2a8194575ccf6d, server=448ea6c54a8c,35099,1732101264437 in 243 msec 2024-11-20T11:14:30,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1e9ad74913c92386ef2a8194575ccf6d, ASSIGN in 1.4770 sec 2024-11-20T11:14:30,600 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0c331e73b55d2ae38913c70b5317c877, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:30,603 INFO [StoreOpener-ab39ce583e7145069013a6bec41b40c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35491,1732101264505 because future has completed 2024-11-20T11:14:30,607 INFO [StoreOpener-ab39ce583e7145069013a6bec41b40c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab39ce583e7145069013a6bec41b40c0 columnFamilyName family 2024-11-20T11:14:30,607 DEBUG [StoreOpener-ab39ce583e7145069013a6bec41b40c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:30,608 INFO [StoreOpener-ab39ce583e7145069013a6bec41b40c0-1 {}] regionserver.HStore(327): Store=ab39ce583e7145069013a6bec41b40c0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:30,608 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,610 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,611 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,612 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,612 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=7 2024-11-20T11:14:30,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35491,1732101264505 in 248 msec 2024-11-20T11:14:30,615 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN in 1.4960 sec 2024-11-20T11:14:30,641 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ab39ce583e7145069013a6bec41b40c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T11:14:30,642 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened ab39ce583e7145069013a6bec41b40c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63706661, jitterRate=-0.05069677531719208}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:30,642 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:30,642 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for ab39ce583e7145069013a6bec41b40c0: Running coprocessor pre-open hook at 1732101270569Writing region info on filesystem at 1732101270569Initializing all the Stores at 1732101270572 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101270572Cleaning up temporary data from old regions at 1732101270612 (+40 ms)Running coprocessor post-open hooks at 1732101270642 (+30 ms)Region opened successfully at 1732101270642 2024-11-20T11:14:30,644 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0., pid=56, masterSystemTime=1732101270496 2024-11-20T11:14:30,648 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:30,648 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:30,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, openSeqNum=2, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:30,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab39ce583e7145069013a6bec41b40c0, server=448ea6c54a8c,43507,1732101264298 because future has completed 2024-11-20T11:14:30,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-20T11:14:30,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure ab39ce583e7145069013a6bec41b40c0, server=448ea6c54a8c,43507,1732101264298 in 291 msec 2024-11-20T11:14:30,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=4 2024-11-20T11:14:30,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ab39ce583e7145069013a6bec41b40c0, ASSIGN in 1.5410 sec 2024-11-20T11:14:30,666 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T11:14:30,667 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732101270666"}]},"ts":"1732101270666"} 2024-11-20T11:14:30,671 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-20T11:14:30,674 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T11:14:30,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 2.8200 sec 2024-11-20T11:14:30,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741829_1005 (size=34) 2024-11-20T11:14:30,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741828_1004 (size=1189) 2024-11-20T11:14:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T11:14:32,073 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-20T11:14:32,073 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-20T11:14:32,074 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T11:14:32,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-20T11:14:32,097 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T11:14:32,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-20T11:14:32,099 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T11:14:32,105 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T11:14:32,133 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T11:14:32,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T11:14:32,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T11:14:32,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T11:14:32,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T11:14:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1e9ad74913c92386ef2a8194575ccf6d changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bac309a4ff803986756ba76688d03707 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ddf1f8c8ecace91d120dbb68cbf18fb1 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 70c2f19bb4361dae17cea153d84e898a changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ec000da7633d9b0b0b955ae7aa7e962f changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 031ddda36eb9bb40423f7c9f4322100c changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 171d75c7056fb7f3c57a7e0ca3724f6d changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 16717e829fb199e0c26298a5116a0b4e changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fb17647ae131913beb1233c85f78d4dc changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0c331e73b55d2ae38913c70b5317c877 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1a244f5f504c74b0b1dff6963a37aff0 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 073880d856b42f096b9b41dfd0b349af changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d221d096301219259fde4786f6bb7402 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c8b0425cde259974fdc42bd0afa1e9d6 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 10e6b7dac8e72b3715b09f737ea5dce3 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2b5da0de9b95a9822f9526f93d4d7af1 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 400f5d1df45ccf5378a59a8cd78bb9d8 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ab39ce583e7145069013a6bec41b40c0 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cb4960297e33c5789bed6704818bf798 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 19c7dd4cb6560dd941c762414bfe9afa changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4e8a8778f20ffb0d01039b06a807ee6a changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 94ab7bd7c2c2041d4db8eb8309fa9cea changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6864708d24f9dd54d37dd8e832ca6cb9 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 07feff3ab7eab5622741a0364ece2464 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 349e25c47fdd7d8457f465670572f372 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 53a2311878635447fc5975caced0a7d4 changed from -1.0 to 0.0, refreshing cache 2024-11-20T11:14:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-20T11:14:32,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-20T11:14:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-20T11:14:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(204): Hosts are {448ea6c54a8c=0} racks are {/default-rack=0} 2024-11-20T11:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-20T11:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-20T11:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-20T11:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T11:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T11:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T11:14:32,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T11:14:32,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T11:14:32,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T11:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-20T11:14:32,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-20T11:14:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-20T11:14:32,276 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T11:14:32,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-20T11:14:32,346 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=639, MaxFileDescriptor=1048576, SystemLoadAverage=453, ProcessCount=11, AvailableMemoryMB=8586 2024-11-20T11:14:32,352 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-20T11:14:32,353 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-20T11:14:32,353 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:32,354 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 448ea6c54a8c,35491,1732101264505: Simulated kill ***** 2024-11-20T11:14:32,354 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T11:14:32,358 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T11:14:32,361 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T11:14:32,362 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T11:14:32,362 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T11:14:32,362 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 318943896 }, "NonHeapMemoryUsage": { "committed": 153485312, "init": 7667712, "max": -1, "used": 151108192 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "448ea6c54a8c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "448ea6c54a8c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "448ea6c54a8c" } ] } 2024-11-20T11:14:32,366 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38459 {}] master.MasterRpcServices(700): 448ea6c54a8c,35491,1732101264505 reported a fatal error: ***** ABORTING region server 448ea6c54a8c,35491,1732101264505: Simulated kill ***** 2024-11-20T11:14:32,370 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '448ea6c54a8c,35491,1732101264505' ***** 2024-11-20T11:14:32,370 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-20T11:14:32,370 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T11:14:32,371 INFO [RS:2;448ea6c54a8c:35491 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T11:14:32,371 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T11:14:32,371 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T11:14:32,371 INFO [RS:2;448ea6c54a8c:35491 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T11:14:32,371 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T11:14:32,371 INFO [RS:2;448ea6c54a8c:35491 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;448ea6c54a8c:35491. 2024-11-20T11:14:32,371 DEBUG [RS:2;448ea6c54a8c:35491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T11:14:32,384 DEBUG [RS:2;448ea6c54a8c:35491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:32,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741836_1012 (size=32114) 2024-11-20T11:14:32,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741836_1012 (size=32114) 2024-11-20T11:14:32,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741836_1012 (size=32114) 2024-11-20T11:14:32,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741832_1008 (size=2237) 2024-11-20T11:14:32,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741832_1008 (size=2237) 2024-11-20T11:14:32,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741832_1008 (size=2237) 2024-11-20T11:14:32,403 DEBUG [RS:2;448ea6c54a8c:35491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:32,403 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T11:14:32,404 INFO [RS:2;448ea6c54a8c:35491 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35491 2024-11-20T11:14:32,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/448ea6c54a8c,35491,1732101264505 2024-11-20T11:14:32,423 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T11:14:32,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T11:14:32,429 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [448ea6c54a8c,35491,1732101264505] 2024-11-20T11:14:32,431 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/448ea6c54a8c,35491,1732101264505 already deleted, retry=false 2024-11-20T11:14:32,432 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 448ea6c54a8c,35491,1732101264505 on 448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:32,439 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:32,446 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:32,449 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true, isMeta: true 2024-11-20T11:14:32,449 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=57 for 448ea6c54a8c,35491,1732101264505 (carryingMeta=true) 448ea6c54a8c,35491,1732101264505/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4fee8015[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T11:14:32,450 INFO [regionserver/448ea6c54a8c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T11:14:32,457 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting 2024-11-20T11:14:32,459 INFO [PEWorker-1 {}] master.SplitWALManager(105): 448ea6c54a8c,35491,1732101264505 WAL count=1, meta=true 2024-11-20T11:14:32,472 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T11:14:32,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta}] 2024-11-20T11:14:32,484 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:32,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:32,514 INFO [regionserver/448ea6c54a8c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T11:14:32,515 INFO [regionserver/448ea6c54a8c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T11:14:32,529 INFO [RS:2;448ea6c54a8c:35491 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T11:14:32,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T11:14:32,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35491-0x1013515b6630003, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T11:14:32,529 INFO [RS:2;448ea6c54a8c:35491 {}] regionserver.HRegionServer(1031): Exiting; stopping=448ea6c54a8c,35491,1732101264505; zookeeper connection closed. 2024-11-20T11:14:32,530 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7066bbd3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7066bbd3 2024-11-20T11:14:32,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35099 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-20T11:14:32,687 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, size=31.4 K (32114bytes) 2024-11-20T11:14:32,688 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T11:14:32,688 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T11:14:32,688 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta 2024-11-20T11:14:32,695 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta after 5ms 2024-11-20T11:14:32,705 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta took 20ms 2024-11-20T11:14:32,717 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T11:14:32,725 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta.temp 2024-11-20T11:14:32,727 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta so closing down 2024-11-20T11:14:32,727 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T11:14:32,729 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta.temp 2024-11-20T11:14:32,733 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T11:14:32,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741864_1040 (size=31873) 2024-11-20T11:14:32,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741864_1040 (size=31873) 2024-11-20T11:14:32,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741864_1040 (size=31873) 2024-11-20T11:14:32,753 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-20T11:14:32,756 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta.temp to hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-20T11:14:32,760 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 49 ms; skipped=1; WAL=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-20T11:14:32,761 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, journal: Splitting hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, size=31.4 K (32114bytes) at 1732101272687Finishing writing output for hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta so closing down at 1732101272727 (+40 ms)Creating recovered edits writer path=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta.temp at 1732101272729 (+2 ms)3 split writer threads finished at 1732101272734 (+5 ms)Closed recovered edits writer path=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1732101272753 (+19 ms)Rename recovered edits hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta.temp to hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732101272756 (+3 ms)Processed 58 edits across 1 Regions in 49 ms; skipped=1; WAL=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732101272760 (+4 ms) 2024-11-20T11:14:32,761 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta 2024-11-20T11:14:32,763 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-20T11:14:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-20T11:14:32,773 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta to hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs 2024-11-20T11:14:32,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T11:14:32,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 in 285 msec 2024-11-20T11:14:32,778 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:32,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-20T11:14:32,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 in 318 msec 2024-11-20T11:14:32,785 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting dir is empty, no logs to split. 2024-11-20T11:14:32,785 INFO [PEWorker-1 {}] master.SplitWALManager(105): 448ea6c54a8c,35491,1732101264505 WAL count=0, meta=true 2024-11-20T11:14:32,785 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 448ea6c54a8c,35491,1732101264505 WAL splitting is done? wals=0, meta=true 2024-11-20T11:14:32,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T11:14:32,792 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:32,794 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:32,895 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T11:14:32,945 DEBUG [448ea6c54a8c:38459 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T11:14:32,945 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(204): Hosts are {448ea6c54a8c=0} racks are {/default-rack=0} 2024-11-20T11:14:32,945 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T11:14:32,945 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T11:14:32,945 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T11:14:32,945 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T11:14:32,945 INFO [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T11:14:32,945 INFO [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T11:14:32,945 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T11:14:32,946 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:32,948 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPENING 2024-11-20T11:14:32,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:32,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:32,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:32,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:32,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:32,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:32,954 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:32,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:32,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-20T11:14:32,979 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T11:14:33,113 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T11:14:33,113 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T11:14:33,114 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T11:14:33,116 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=448ea6c54a8c%2C43507%2C1732101264298.meta, suffix=.meta, logDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,43507,1732101264298, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs, maxLogs=32 2024-11-20T11:14:33,131 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,43507,1732101264298/448ea6c54a8c%2C43507%2C1732101264298.meta.1732101273117.meta, exclude list is [], retry=0 2024-11-20T11:14:33,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37351,DS-dfc6e092-f541-494b-81e4-826b95dc8a56,DISK] 2024-11-20T11:14:33,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44787,DS-5f4458a9-3703-4eaa-8233-a851e4aaf890,DISK] 2024-11-20T11:14:33,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42077,DS-ef273f42-ddeb-44b0-90ec-87c34e9d8141,DISK] 2024-11-20T11:14:33,137 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,43507,1732101264298/448ea6c54a8c%2C43507%2C1732101264298.meta.1732101273117.meta 2024-11-20T11:14:33,137 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:44047:44047),(127.0.0.1/127.0.0.1:40685:40685)] 2024-11-20T11:14:33,137 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T11:14:33,138 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T11:14:33,138 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T11:14:33,138 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T11:14:33,138 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T11:14:33,138 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:33,139 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T11:14:33,139 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T11:14:33,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T11:14:33,142 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T11:14:33,142 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:33,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:33,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T11:14:33,144 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T11:14:33,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:33,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:33,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T11:14:33,145 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T11:14:33,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:33,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:33,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T11:14:33,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T11:14:33,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:33,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T11:14:33,148 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T11:14:33,149 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740 2024-11-20T11:14:33,151 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740 2024-11-20T11:14:33,152 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-20T11:14:33,171 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-20T11:14:33,175 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-20T11:14:33,252 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/info/240bd87f51004c82b01f23186355c948 is 138, key is test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0./info:regioninfo/1732101270649/Put/seqid=0 2024-11-20T11:14:33,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741866_1042 (size=34924) 2024-11-20T11:14:33,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741866_1042 (size=34924) 2024-11-20T11:14:33,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741866_1042 (size=34924) 2024-11-20T11:14:33,269 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/info/240bd87f51004c82b01f23186355c948 2024-11-20T11:14:33,341 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/ns/f5a7c64c284a4192accc1e3e498dfb2b is 43, key is default/ns:d/1732101267536/Put/seqid=0 2024-11-20T11:14:33,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741867_1043 (size=5153) 2024-11-20T11:14:33,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741867_1043 (size=5153) 2024-11-20T11:14:33,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741867_1043 (size=5153) 2024-11-20T11:14:33,352 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/ns/f5a7c64c284a4192accc1e3e498dfb2b 2024-11-20T11:14:33,376 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/table/e3e94500b87848fb927bd7f81776f99a is 40, key is test/table:state/1732101270666/Put/seqid=0 2024-11-20T11:14:33,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741868_1044 (size=5165) 2024-11-20T11:14:33,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741868_1044 (size=5165) 2024-11-20T11:14:33,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741868_1044 (size=5165) 2024-11-20T11:14:33,386 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/table/e3e94500b87848fb927bd7f81776f99a 2024-11-20T11:14:33,394 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/info/240bd87f51004c82b01f23186355c948 as hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/info/240bd87f51004c82b01f23186355c948 2024-11-20T11:14:33,402 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/info/240bd87f51004c82b01f23186355c948, entries=260, sequenceid=60, filesize=34.1 K 2024-11-20T11:14:33,405 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/ns/f5a7c64c284a4192accc1e3e498dfb2b as hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/ns/f5a7c64c284a4192accc1e3e498dfb2b 2024-11-20T11:14:33,413 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/ns/f5a7c64c284a4192accc1e3e498dfb2b, entries=2, sequenceid=60, filesize=5.0 K 2024-11-20T11:14:33,414 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/table/e3e94500b87848fb927bd7f81776f99a as hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/table/e3e94500b87848fb927bd7f81776f99a 2024-11-20T11:14:33,422 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/table/e3e94500b87848fb927bd7f81776f99a, entries=2, sequenceid=60, filesize=5.0 K 2024-11-20T11:14:33,428 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 250ms, sequenceid=60, compaction requested=false; wal=null 2024-11-20T11:14:33,430 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-20T11:14:33,432 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T11:14:33,432 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T11:14:33,433 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T11:14:33,435 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T11:14:33,439 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-20T11:14:33,441 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74429922, jitterRate=0.10909226536750793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T11:14:33,441 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T11:14:33,441 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732101273139Writing region info on filesystem at 1732101273139Initializing all the Stores at 1732101273140 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101273140Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101273140Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101273141 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732101273141Obtaining lock to block concurrent updates at 1732101273176 (+35 ms)Preparing flush snapshotting stores in 1588230740 at 1732101273176Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732101273179 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732101273179Flushing 1588230740/info: creating writer at 1732101273181 (+2 ms)Flushing 1588230740/info: appending metadata at 1732101273240 (+59 ms)Flushing 1588230740/info: closing flushed file at 1732101273244 (+4 ms)Flushing 1588230740/ns: creating writer at 1732101273324 (+80 ms)Flushing 1588230740/ns: appending metadata at 1732101273340 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732101273340Flushing 1588230740/table: creating writer at 1732101273360 (+20 ms)Flushing 1588230740/table: appending metadata at 1732101273376 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732101273376Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67886733: reopening flushed file at 1732101273393 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@583fc279: reopening flushed file at 1732101273404 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a23c0d2: reopening flushed file at 1732101273413 (+9 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 250ms, sequenceid=60, compaction requested=false; wal=null at 1732101273428 (+15 ms)Cleaning up temporary data from old regions at 1732101273432 (+4 ms)Running coprocessor post-open hooks at 1732101273441 (+9 ms)Region opened successfully at 1732101273441 2024-11-20T11:14:33,443 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732101273108 2024-11-20T11:14:33,445 DEBUG [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T11:14:33,445 INFO [RS_OPEN_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T11:14:33,446 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:33,447 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:33,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:33,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:33,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:33,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:33,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:33,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:33,451 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:33,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T11:14:33,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 in 497 msec 2024-11-20T11:14:33,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-20T11:14:33,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 666 msec 2024-11-20T11:14:33,458 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 448ea6c54a8c,35491,1732101264505 had 9 regions 2024-11-20T11:14:33,459 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true, isMeta: false 2024-11-20T11:14:33,461 INFO [PEWorker-1 {}] master.SplitWALManager(105): 448ea6c54a8c,35491,1732101264505 WAL count=1, meta=false 2024-11-20T11:14:33,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706}] 2024-11-20T11:14:33,463 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:33,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:33,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35099 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-20T11:14:33,634 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706, size=2.2 K (2237bytes) 2024-11-20T11:14:33,635 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706 2024-11-20T11:14:33,635 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706 after 0ms 2024-11-20T11:14:33,638 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706 took 4ms 2024-11-20T11:14:33,641 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 073880d856b42f096b9b41dfd0b349af: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T11:14:33,643 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1a244f5f504c74b0b1dff6963a37aff0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T11:14:33,644 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2b5da0de9b95a9822f9526f93d4d7af1: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T11:14:33,646 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d221d096301219259fde4786f6bb7402: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T11:14:33,647 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 400f5d1df45ccf5378a59a8cd78bb9d8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T11:14:33,649 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 10e6b7dac8e72b3715b09f737ea5dce3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T11:14:33,650 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c8b0425cde259974fdc42bd0afa1e9d6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T11:14:33,651 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 0c331e73b55d2ae38913c70b5317c877: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T11:14:33,652 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706 so closing down 2024-11-20T11:14:33,652 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T11:14:33,652 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T11:14:33,652 INFO [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 13 ms; skipped=8; WAL=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-20T11:14:33,652 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706, journal: Splitting hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706, size=2.2 K (2237bytes) at 1732101273635Finishing writing output for hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706 so closing down at 1732101273652 (+17 ms)3 split writer threads finished at 1732101273652Processed 8 edits across 0 Regions in 13 ms; skipped=8; WAL=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732101273652 2024-11-20T11:14:33,652 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706 2024-11-20T11:14:33,652 DEBUG [RS_LOG_REPLAY_OPS-regionserver/448ea6c54a8c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-20T11:14:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38459 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-20T11:14:33,657 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting/448ea6c54a8c%2C35491%2C1732101264505.1732101266706 to hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs 2024-11-20T11:14:33,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-20T11:14:33,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 in 193 msec 2024-11-20T11:14:33,661 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:33,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-20T11:14:33,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 in 200 msec 2024-11-20T11:14:33,665 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/WALs/448ea6c54a8c,35491,1732101264505-splitting dir is empty, no logs to split. 2024-11-20T11:14:33,665 INFO [PEWorker-1 {}] master.SplitWALManager(105): 448ea6c54a8c,35491,1732101264505 WAL count=0, meta=false 2024-11-20T11:14:33,665 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 448ea6c54a8c,35491,1732101264505 WAL splitting is done? wals=0, meta=false 2024-11-20T11:14:33,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN}] 2024-11-20T11:14:33,669 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:33,670 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:33,671 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:33,672 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:33,672 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:33,673 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:33,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:33,676 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:33,676 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:33,677 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:33,678 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:33,679 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:33,680 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:33,680 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:33,681 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:33,682 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T11:14:33,820 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(204): Hosts are {448ea6c54a8c=0} racks are {/default-rack=0} 2024-11-20T11:14:33,821 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T11:14:33,821 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T11:14:33,821 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T11:14:33,821 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T11:14:33,821 INFO [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T11:14:33,821 INFO [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T11:14:33,821 DEBUG [448ea6c54a8c:38459 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T11:14:33,821 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:33,822 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:33,823 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:33,823 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 448ea6c54a8c:35491 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T11:14:33,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, regionLocation=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:33,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1, error=java.net.ConnectException: Call to address=448ea6c54a8c:35491 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 2024-11-20T11:14:33,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:33,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T11:14:33,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1 from cache 2024-11-20T11:14:33,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:33,825 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T11:14:33,825 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,43507,1732101264298, seqNum=-1] 2024-11-20T11:14:33,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.FailedServers(52): Added failed server with address 448ea6c54a8c:35491 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 2024-11-20T11:14:33,826 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T11:14:33,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=d221d096301219259fde4786f6bb7402, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:33,826 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 448ea6c54a8c:35491 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T11:14:33,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,43507,1732101264298, seqNum=-1, error=java.net.ConnectException: Call to address=448ea6c54a8c:35491 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 2024-11-20T11:14:33,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,43507,1732101264298, seqNum=-1, error=java.net.ConnectException: Call to address=448ea6c54a8c:35491 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 2024-11-20T11:14:33,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, regionLocation=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:33,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,43507,1732101264298, seqNum=-1, error=java.net.ConnectException: Call to address=448ea6c54a8c:35491 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 2024-11-20T11:14:33,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,35491,1732101264505, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=448ea6c54a8c,43507,1732101264298, seqNum=-1, error=java.net.ConnectException: Call to address=448ea6c54a8c:35491 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 2024-11-20T11:14:33,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.FailedServers(52): Added failed server with address 448ea6c54a8c:35491 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 448ea6c54a8c/172.17.0.2:35491 2024-11-20T11:14:33,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60177, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T11:14:33,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN because future has completed 2024-11-20T11:14:33,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:33,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN because future has completed 2024-11-20T11:14:33,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN because future has completed 2024-11-20T11:14:33,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:33,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:33,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T11:14:33,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T11:14:33,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN because future has completed 2024-11-20T11:14:33,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:33,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN because future has completed 2024-11-20T11:14:33,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-20T11:14:33,935 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-20T11:14:33,936 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T11:14:33,936 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T11:14:33,937 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T11:14:33,937 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T11:14:33,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:33,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN because future has completed 2024-11-20T11:14:33,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN because future has completed 2024-11-20T11:14:33,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN because future has completed 2024-11-20T11:14:33,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298}] 2024-11-20T11:14:33,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:33,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 073880d856b42f096b9b41dfd0b349af, server=448ea6c54a8c,35099,1732101264437}] 2024-11-20T11:14:33,988 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:33,988 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a244f5f504c74b0b1dff6963a37aff0, NAME => 'test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-20T11:14:33,988 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,988 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:33,988 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,988 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,990 INFO [StoreOpener-1a244f5f504c74b0b1dff6963a37aff0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,991 INFO [StoreOpener-1a244f5f504c74b0b1dff6963a37aff0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a244f5f504c74b0b1dff6963a37aff0 columnFamilyName family 2024-11-20T11:14:33,991 DEBUG [StoreOpener-1a244f5f504c74b0b1dff6963a37aff0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:33,992 INFO [StoreOpener-1a244f5f504c74b0b1dff6963a37aff0-1 {}] regionserver.HStore(327): Store=1a244f5f504c74b0b1dff6963a37aff0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:33,992 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,992 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,994 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,994 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,994 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,996 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,997 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 1a244f5f504c74b0b1dff6963a37aff0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64104511, jitterRate=-0.04476834833621979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:33,997 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:33,997 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 1a244f5f504c74b0b1dff6963a37aff0: Running coprocessor pre-open hook at 1732101273989Writing region info on filesystem at 1732101273989Initializing all the Stores at 1732101273989Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101273990 (+1 ms)Cleaning up temporary data from old regions at 1732101273994 (+4 ms)Running coprocessor post-open hooks at 1732101273997 (+3 ms)Region opened successfully at 1732101273997 2024-11-20T11:14:33,998 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0., pid=73, masterSystemTime=1732101273983 2024-11-20T11:14:34,000 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:34,000 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:34,000 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:34,001 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => d221d096301219259fde4786f6bb7402, NAME => 'test,lll,1732101267801.d221d096301219259fde4786f6bb7402.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-20T11:14:34,001 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,001 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,lll,1732101267801.d221d096301219259fde4786f6bb7402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:34,001 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,001 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,003 INFO [StoreOpener-d221d096301219259fde4786f6bb7402-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,003 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-20T11:14:34,003 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,004 INFO [StoreOpener-d221d096301219259fde4786f6bb7402-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d221d096301219259fde4786f6bb7402 columnFamilyName family 2024-11-20T11:14:34,004 DEBUG [StoreOpener-d221d096301219259fde4786f6bb7402-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:34,004 INFO [StoreOpener-d221d096301219259fde4786f6bb7402-1 {}] regionserver.HStore(327): Store=d221d096301219259fde4786f6bb7402/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:34,004 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,003 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,005 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,006 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,006 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,007 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,008 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,009 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened d221d096301219259fde4786f6bb7402; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71902624, jitterRate=0.07143259048461914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:34,010 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,010 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for d221d096301219259fde4786f6bb7402: Running coprocessor pre-open hook at 1732101274001Writing region info on filesystem at 1732101274001Initializing all the Stores at 1732101274002 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101274002Cleaning up temporary data from old regions at 1732101274007 (+5 ms)Running coprocessor post-open hooks at 1732101274010 (+3 ms)Region opened successfully at 1732101274010 2024-11-20T11:14:34,010 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732101267801.d221d096301219259fde4786f6bb7402., pid=74, masterSystemTime=1732101273983 2024-11-20T11:14:34,013 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:34,013 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:34,013 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:34,013 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 400f5d1df45ccf5378a59a8cd78bb9d8, NAME => 'test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-20T11:14:34,013 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,014 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:34,014 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,014 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,013 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-20T11:14:34,015 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,015 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 073880d856b42f096b9b41dfd0b349af, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,015 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 073880d856b42f096b9b41dfd0b349af, server=448ea6c54a8c,35099,1732101264437 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,016 INFO [StoreOpener-400f5d1df45ccf5378a59a8cd78bb9d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,017 INFO [StoreOpener-400f5d1df45ccf5378a59a8cd78bb9d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 400f5d1df45ccf5378a59a8cd78bb9d8 columnFamilyName family 2024-11-20T11:14:34,017 DEBUG [StoreOpener-400f5d1df45ccf5378a59a8cd78bb9d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:34,018 INFO [StoreOpener-400f5d1df45ccf5378a59a8cd78bb9d8-1 {}] regionserver.HStore(327): Store=400f5d1df45ccf5378a59a8cd78bb9d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:34,018 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,019 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,020 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,020 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,021 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,022 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,023 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 400f5d1df45ccf5378a59a8cd78bb9d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72349534, jitterRate=0.0780920684337616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:34,023 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,024 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 400f5d1df45ccf5378a59a8cd78bb9d8: Running coprocessor pre-open hook at 1732101274014Writing region info on filesystem at 1732101274014Initializing all the Stores at 1732101274015 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101274015Cleaning up temporary data from old regions at 1732101274021 (+6 ms)Running coprocessor post-open hooks at 1732101274024 (+3 ms)Region opened successfully at 1732101274024 2024-11-20T11:14:34,024 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8., pid=77, masterSystemTime=1732101273983 2024-11-20T11:14:34,027 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:34,027 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:34,027 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:34,027 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 10e6b7dac8e72b3715b09f737ea5dce3, NAME => 'test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-20T11:14:34,027 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,027 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:34,028 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,028 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,029 INFO [StoreOpener-10e6b7dac8e72b3715b09f737ea5dce3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,030 INFO [StoreOpener-10e6b7dac8e72b3715b09f737ea5dce3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10e6b7dac8e72b3715b09f737ea5dce3 columnFamilyName family 2024-11-20T11:14:34,030 DEBUG [StoreOpener-10e6b7dac8e72b3715b09f737ea5dce3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:34,031 INFO [StoreOpener-10e6b7dac8e72b3715b09f737ea5dce3-1 {}] regionserver.HStore(327): Store=10e6b7dac8e72b3715b09f737ea5dce3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:34,031 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,032 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,033 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,033 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,033 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,035 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,036 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 10e6b7dac8e72b3715b09f737ea5dce3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67736450, jitterRate=0.009351760149002075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:34,036 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,036 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 10e6b7dac8e72b3715b09f737ea5dce3: Running coprocessor pre-open hook at 1732101274028Writing region info on filesystem at 1732101274028Initializing all the Stores at 1732101274029 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101274029Cleaning up temporary data from old regions at 1732101274033 (+4 ms)Running coprocessor post-open hooks at 1732101274036 (+3 ms)Region opened successfully at 1732101274036 2024-11-20T11:14:34,037 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3., pid=72, masterSystemTime=1732101273983 2024-11-20T11:14:34,040 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:34,040 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:34,076 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,076 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,076 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,076 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,077 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,077 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,077 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,077 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,077 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,078 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,083 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,084 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,085 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 073880d856b42f096b9b41dfd0b349af, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,091 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,092 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 073880d856b42f096b9b41dfd0b349af, NAME => 'test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,092 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,092 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,092 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,092 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,092 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,092 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,092 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,092 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,092 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,092 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,093 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,093 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 073880d856b42f096b9b41dfd0b349af, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,093 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-20T11:14:34,094 INFO [StoreOpener-073880d856b42f096b9b41dfd0b349af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,095 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,096 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,097 INFO [StoreOpener-073880d856b42f096b9b41dfd0b349af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 073880d856b42f096b9b41dfd0b349af columnFamilyName family 2024-11-20T11:14:34,097 DEBUG [StoreOpener-073880d856b42f096b9b41dfd0b349af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T11:14:34,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,101 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304101 2024-11-20T11:14:34,101 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,102 INFO [StoreOpener-073880d856b42f096b9b41dfd0b349af-1 {}] regionserver.HStore(327): Store=073880d856b42f096b9b41dfd0b349af/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T11:14:34,102 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,103 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,103 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,105 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,107 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,107 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,107 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,109 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,109 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,109 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35099,1732101264437 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,110 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,111 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 073880d856b42f096b9b41dfd0b349af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63618316, jitterRate=-0.05201321840286255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T11:14:34,111 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 073880d856b42f096b9b41dfd0b349af 2024-11-20T11:14:34,112 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 073880d856b42f096b9b41dfd0b349af: Running coprocessor pre-open hook at 1732101274093Writing region info on filesystem at 1732101274093Initializing all the Stores at 1732101274094 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732101274094Cleaning up temporary data from old regions at 1732101274107 (+13 ms)Running coprocessor post-open hooks at 1732101274111 (+4 ms)Region opened successfully at 1732101274111 2024-11-20T11:14:34,113 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af., pid=79, masterSystemTime=1732101274086 2024-11-20T11:14:34,114 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1916) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1280) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1256) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,118 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.CallRunner(138): callId: 92 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 126 connection: 172.17.0.2:46291 deadline: 1732101284113, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-11-20T11:14:34,120 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "448ea6c54a8c" port: 35099 start_code: 1732101264437 } transition { transition_code: OPENED region_info { region_id: 1732101267801 table_name { namespace: "default" qualifier: "test" } start_key: "jjj" end_key: "kkk" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732101264633 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1916) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1280) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1206) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1256) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1916) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1280) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1206) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1256) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-20T11:14:34,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,136 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334136 2024-11-20T11:14:34,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394136 2024-11-20T11:14:34,136 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 34 msec 2024-11-20T11:14:34,136 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,136 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,136 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,137 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,137 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,137 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,138 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,145 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,145 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,145 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,145 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,145 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,145 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,151 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,151 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,151 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,152 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,152 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,152 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,152 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,152 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,152 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,152 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c331e73b55d2ae38913c70b5317c877, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-20T11:14:34,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,154 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,154 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,157 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,157 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,157 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,157 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,158 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,158 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,158 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,158 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,158 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,158 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,158 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,158 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304158 2024-11-20T11:14:34,158 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,158 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,160 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,161 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,161 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,162 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,171 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334171 2024-11-20T11:14:34,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394171 2024-11-20T11:14:34,171 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-20T11:14:34,171 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,171 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,171 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,171 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,172 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,172 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,172 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,172 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,173 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,179 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,180 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,180 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 400f5d1df45ccf5378a59a8cd78bb9d8, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,181 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T11:14:34,181 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,182 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,183 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,186 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,188 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304188 2024-11-20T11:14:34,188 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,188 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,191 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,200 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334200 2024-11-20T11:14:34,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394200 2024-11-20T11:14:34,200 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-20T11:14:34,200 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,200 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,200 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,201 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,201 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,201 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,202 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8b0425cde259974fdc42bd0afa1e9d6, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,209 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T11:14:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,211 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,211 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304215 2024-11-20T11:14:34,215 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,215 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,217 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,219 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,219 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,219 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,226 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334226 2024-11-20T11:14:34,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394226 2024-11-20T11:14:34,226 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-20T11:14:34,226 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,226 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,226 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,226 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,227 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,227 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b5da0de9b95a9822f9526f93d4d7af1, server=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,235 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T11:14:34,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,236 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,237 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,240 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,240 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,241 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,241 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,241 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,241 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,241 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,241 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,241 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304241 2024-11-20T11:14:34,241 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,241 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,243 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,244 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,245 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,245 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,255 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334255 2024-11-20T11:14:34,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394255 2024-11-20T11:14:34,255 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-20T11:14:34,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,255 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,255 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,255 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,256 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,260 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d221d096301219259fde4786f6bb7402, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,266 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T11:14:34,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,269 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,269 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,274 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304274 2024-11-20T11:14:34,274 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,274 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,276 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,278 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,278 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,278 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,287 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,288 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334288 2024-11-20T11:14:34,288 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394288 2024-11-20T11:14:34,288 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-20T11:14:34,288 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,288 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,288 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,288 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,288 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,288 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,288 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,289 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,291 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T11:14:34,291 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,299 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,299 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-20T11:14:34,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,301 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,301 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,306 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,306 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,306 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,306 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,306 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,306 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,306 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,306 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,312 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304312 2024-11-20T11:14:34,312 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,313 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,317 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,325 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,325 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334325 2024-11-20T11:14:34,326 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394325 2024-11-20T11:14:34,326 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-20T11:14:34,326 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,326 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,326 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,326 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,327 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,327 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,327 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,336 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,336 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,336 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e6b7dac8e72b3715b09f737ea5dce3, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,337 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T11:14:34,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,339 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,340 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,344 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,344 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304344 2024-11-20T11:14:34,344 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,345 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,346 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,348 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,348 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,348 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,355 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,355 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334355 2024-11-20T11:14:34,355 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394355 2024-11-20T11:14:34,356 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-20T11:14:34,356 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,356 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,356 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,356 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,356 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,356 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,356 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,356 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,357 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,361 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,364 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,364 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN 2024-11-20T11:14:34,365 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T11:14:34,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,369 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,369 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,381 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,382 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,382 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,382 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,382 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,382 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8b0425cde259974fdc42bd0afa1e9d6, ASSIGN to state=OFFLINE, location=null, table=test, region=c8b0425cde259974fdc42bd0afa1e9d6 to restore RIT 2024-11-20T11:14:34,391 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304391 2024-11-20T11:14:34,391 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,392 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,396 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,398 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,398 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,399 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,401 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,401 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334401 2024-11-20T11:14:34,401 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394401 2024-11-20T11:14:34,401 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-20T11:14:34,401 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,401 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,401 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,402 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,402 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,403 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,410 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,410 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,410 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,410 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,410 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,410 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,411 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,411 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,411 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN 2024-11-20T11:14:34,411 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T11:14:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,412 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,412 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,421 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,421 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,421 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,421 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,421 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,421 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,421 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d221d096301219259fde4786f6bb7402, ASSIGN to state=OFFLINE, location=null, table=test, region=d221d096301219259fde4786f6bb7402 to restore RIT 2024-11-20T11:14:34,426 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304426 2024-11-20T11:14:34,426 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,427 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,429 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,430 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,430 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,430 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,436 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,436 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334436 2024-11-20T11:14:34,437 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394436 2024-11-20T11:14:34,437 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-20T11:14:34,437 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,437 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,437 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,437 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,437 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,437 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,437 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,438 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,439 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,444 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,445 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,445 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,445 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,445 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,445 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,445 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,445 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,445 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,445 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,445 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,445 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN 2024-11-20T11:14:34,445 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T11:14:34,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,446 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,446 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,452 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,453 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,453 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,453 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,453 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,453 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,453 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10e6b7dac8e72b3715b09f737ea5dce3, ASSIGN to state=OFFLINE, location=null, table=test, region=10e6b7dac8e72b3715b09f737ea5dce3 to restore RIT 2024-11-20T11:14:34,453 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304453 2024-11-20T11:14:34,454 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,454 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,456 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,460 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,460 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,461 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,464 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334464 2024-11-20T11:14:34,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394464 2024-11-20T11:14:34,464 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-20T11:14:34,464 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,464 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,464 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,464 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,464 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,465 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,465 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,471 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,471 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,471 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,471 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,471 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,471 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN 2024-11-20T11:14:34,472 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T11:14:34,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,472 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,473 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,475 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,475 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,475 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,477 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,477 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,477 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,477 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,477 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b5da0de9b95a9822f9526f93d4d7af1, ASSIGN to state=OFFLINE, location=null, table=test, region=2b5da0de9b95a9822f9526f93d4d7af1 to restore RIT 2024-11-20T11:14:34,480 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304480 2024-11-20T11:14:34,480 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,480 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,481 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,486 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,486 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,486 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,491 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,491 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334491 2024-11-20T11:14:34,491 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394491 2024-11-20T11:14:34,491 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-20T11:14:34,491 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,491 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,491 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,491 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,491 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,491 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,492 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,492 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,493 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,498 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN 2024-11-20T11:14:34,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T11:14:34,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,499 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,499 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,503 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,503 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,503 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,503 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a244f5f504c74b0b1dff6963a37aff0, ASSIGN to state=OFFLINE, location=null, table=test, region=1a244f5f504c74b0b1dff6963a37aff0 to restore RIT 2024-11-20T11:14:34,507 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304507 2024-11-20T11:14:34,507 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,507 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,509 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,510 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,510 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,510 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,514 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334514 2024-11-20T11:14:34,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394514 2024-11-20T11:14:34,514 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-20T11:14:34,514 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,514 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,514 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,515 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,515 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,515 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,515 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,516 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,519 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,519 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,519 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,519 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN 2024-11-20T11:14:34,520 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T11:14:34,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,520 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,521 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,524 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,524 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,525 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,525 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8, ASSIGN to state=OFFLINE, location=null, table=test, region=400f5d1df45ccf5378a59a8cd78bb9d8 to restore RIT 2024-11-20T11:14:34,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304525 2024-11-20T11:14:34,525 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,525 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,526 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,529 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,535 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334535 2024-11-20T11:14:34,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394535 2024-11-20T11:14:34,535 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-20T11:14:34,535 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,535 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,535 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,535 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,535 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,535 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,535 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,536 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,537 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,540 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,542 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,542 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,542 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN 2024-11-20T11:14:34,542 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T11:14:34,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,543 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,544 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=073880d856b42f096b9b41dfd0b349af, ASSIGN to state=OFFLINE, location=null, table=test, region=073880d856b42f096b9b41dfd0b349af to restore RIT 2024-11-20T11:14:34,552 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304552 2024-11-20T11:14:34,552 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,552 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,554 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,556 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,556 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,556 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,560 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,560 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334560 2024-11-20T11:14:34,560 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394560 2024-11-20T11:14:34,560 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T11:14:34,560 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,560 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,560 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,560 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,560 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,560 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,561 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,561 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,561 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,564 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,565 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,565 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN 2024-11-20T11:14:34,565 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T11:14:34,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,566 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,566 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,569 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0c331e73b55d2ae38913c70b5317c877, ASSIGN to state=OFFLINE, location=null, table=test, region=0c331e73b55d2ae38913c70b5317c877 to restore RIT 2024-11-20T11:14:34,570 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304570 2024-11-20T11:14:34,570 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,570 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,572 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,573 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,573 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,574 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,576 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,577 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334577 2024-11-20T11:14:34,577 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394577 2024-11-20T11:14:34,577 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-20T11:14:34,577 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,577 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,577 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,577 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,577 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,577 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,578 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,579 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,579 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,579 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,579 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,579 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,579 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,579 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,581 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,581 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,581 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,584 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304584 2024-11-20T11:14:34,584 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,585 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,586 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,589 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,589 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,589 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,591 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,591 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334591 2024-11-20T11:14:34,591 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394591 2024-11-20T11:14:34,591 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-20T11:14:34,591 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,591 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,591 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,591 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,591 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,592 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,592 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38459 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.1732101266706, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T11:14:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,597 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,597 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,601 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304601 2024-11-20T11:14:34,601 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,601 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,602 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,605 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,605 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,605 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,608 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,608 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334608 2024-11-20T11:14:34,608 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394608 2024-11-20T11:14:34,608 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-20T11:14:34,608 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,608 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,608 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,608 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,609 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,609 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,609 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,609 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,612 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,612 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T11:14:34,612 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,612 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,613 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,613 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,614 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,614 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,614 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,614 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,614 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=448ea6c54a8c,43507,1732101264298 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,614 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T11:14:34,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,615 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,615 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,619 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,619 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,619 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,620 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304620 2024-11-20T11:14:34,620 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,620 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,621 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,622 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,622 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,622 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,624 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,626 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,626 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334626 2024-11-20T11:14:34,626 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394626 2024-11-20T11:14:34,626 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-20T11:14:34,626 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,626 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,627 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,627 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,627 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,627 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,627 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,627 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,628 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T11:14:34,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,630 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,631 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T11:14:34,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,631 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,631 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,635 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304634 2024-11-20T11:14:34,635 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,635 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,636 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,637 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,637 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,637 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,641 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334641 2024-11-20T11:14:34,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394641 2024-11-20T11:14:34,641 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-20T11:14:34,641 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,641 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,641 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,641 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,641 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,641 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,641 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,641 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,642 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,643 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,643 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T11:14:34,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,645 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T11:14:34,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,646 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,646 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,648 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,648 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,648 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,649 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304649 2024-11-20T11:14:34,649 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,649 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,653 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,653 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,653 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,656 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,656 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334656 2024-11-20T11:14:34,656 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394656 2024-11-20T11:14:34,656 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-20T11:14:34,656 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,656 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,656 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,656 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,656 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,656 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,656 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,657 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,658 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T11:14:34,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 448ea6c54a8c%2C35491%2C1732101264505.meta.1732101267313.meta, worker=448ea6c54a8c,35099,1732101264437 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T11:14:34,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T11:14:34,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,661 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,661 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,663 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,663 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,663 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,664 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304664 2024-11-20T11:14:34,664 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,664 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,666 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,667 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T11:14:34,667 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,667 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,670 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334670 2024-11-20T11:14:34,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394670 2024-11-20T11:14:34,670 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 5 msec 2024-11-20T11:14:34,670 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T11:14:34,670 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,670 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,670 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,671 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T11:14:34,671 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T11:14:34,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T11:14:34,671 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T11:14:34,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T11:14:34,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T11:14:34,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T11:14:34,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T11:14:34,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true 2024-11-20T11:14:34,674 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T11:14:34,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T11:14:34,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=61 2024-11-20T11:14:34,674 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=448ea6c54a8c,43507,1732101264298, table=hbase:meta, region=1588230740 2024-11-20T11:14:34,675 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 448ea6c54a8c,43507,1732101264298, state=OPEN 2024-11-20T11:14:34,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T11:14:34,678 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,678 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,678 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T11:14:34,679 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732101304679 2024-11-20T11:14:34,679 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T11:14:34,679 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T11:14:34,680 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T11:14:34,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1a244f5f504c74b0b1dff6963a37aff0, server=448ea6c54a8c,43507,1732101264298:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 448ea6c54a8c,35491,1732101264505, splitWal=true, meta=true exec-time=2.2470 sec 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e9ad74913c92386ef2a8194575ccf6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab39ce583e7145069013a6bec41b40c0, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0c331e73b55d2ae38913c70b5317c877, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb4960297e33c5789bed6704818bf798, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bac309a4ff803986756ba76688d03707, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a244f5f504c74b0b1dff6963a37aff0, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19c7dd4cb6560dd941c762414bfe9afa, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddf1f8c8ecace91d120dbb68cbf18fb1, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e8a8778f20ffb0d01039b06a807ee6a, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=073880d856b42f096b9b41dfd0b349af, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c2f19bb4361dae17cea153d84e898a, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d221d096301219259fde4786f6bb7402, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94ab7bd7c2c2041d4db8eb8309fa9cea, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8b0425cde259974fdc42bd0afa1e9d6, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec000da7633d9b0b0b955ae7aa7e962f, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=031ddda36eb9bb40423f7c9f4322100c, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=171d75c7056fb7f3c57a7e0ca3724f6d, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6864708d24f9dd54d37dd8e832ca6cb9, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10e6b7dac8e72b3715b09f737ea5dce3, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07feff3ab7eab5622741a0364ece2464, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=16717e829fb199e0c26298a5116a0b4e, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b5da0de9b95a9822f9526f93d4d7af1, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=349e25c47fdd7d8457f465670572f372, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=400f5d1df45ccf5378a59a8cd78bb9d8, regionState=OPENING, lastHost=448ea6c54a8c,35491,1732101264505, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53a2311878635447fc5975caced0a7d4, regionState=OPEN, lastHost=448ea6c54a8c,43507,1732101264298, regionLocation=448ea6c54a8c,43507,1732101264298, openSeqNum=2 2024-11-20T11:14:34,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb17647ae131913beb1233c85f78d4dc, regionState=OPEN, lastHost=448ea6c54a8c,35099,1732101264437, regionLocation=448ea6c54a8c,35099,1732101264437, openSeqNum=2 2024-11-20T11:14:34,685 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T11:14:34,685 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732101334685 2024-11-20T11:14:34,685 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732101394685 2024-11-20T11:14:34,685 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 5 msec 2024-11-20T11:14:34,701 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=346 (was 380), OpenFileDescriptor=616 (was 639), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=453 (was 453), ProcessCount=11 (was 11), AvailableMemoryMB=8529 (was 8586) 2024-11-20T11:14:34,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T11:14:34,702 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T11:14:34,702 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T11:14:34,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:34,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:34,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T11:14:34,703 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T11:14:34,703 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=958468940, stopped=false 2024-11-20T11:14:34,703 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=448ea6c54a8c,38459,1732101263372 2024-11-20T11:14:34,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T11:14:34,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T11:14:34,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:34,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T11:14:34,705 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:34,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:34,706 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T11:14:34,706 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T11:14:34,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:34,706 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T11:14:34,706 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T11:14:34,706 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T11:14:34,706 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '448ea6c54a8c,43507,1732101264298' ***** 2024-11-20T11:14:34,706 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T11:14:34,707 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '448ea6c54a8c,35099,1732101264437' ***** 2024-11-20T11:14:34,707 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T11:14:34,707 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T11:14:34,707 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T11:14:34,707 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T11:14:34,707 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T11:14:34,707 INFO [RS:0;448ea6c54a8c:43507 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T11:14:34,707 INFO [RS:0;448ea6c54a8c:43507 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T11:14:34,707 INFO [RS:1;448ea6c54a8c:35099 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T11:14:34,707 INFO [RS:1;448ea6c54a8c:35099 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T11:14:34,707 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for cb4960297e33c5789bed6704818bf798 2024-11-20T11:14:34,707 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for 031ddda36eb9bb40423f7c9f4322100c 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for 16717e829fb199e0c26298a5116a0b4e 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for ec000da7633d9b0b0b955ae7aa7e962f 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 94ab7bd7c2c2041d4db8eb8309fa9cea 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for ddf1f8c8ecace91d120dbb68cbf18fb1 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 400f5d1df45ccf5378a59a8cd78bb9d8 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for 1e9ad74913c92386ef2a8194575ccf6d 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 19c7dd4cb6560dd941c762414bfe9afa 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for bac309a4ff803986756ba76688d03707 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for ab39ce583e7145069013a6bec41b40c0 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for 171d75c7056fb7f3c57a7e0ca3724f6d 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 4e8a8778f20ffb0d01039b06a807ee6a 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for 70c2f19bb4361dae17cea153d84e898a 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 6864708d24f9dd54d37dd8e832ca6cb9 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 10e6b7dac8e72b3715b09f737ea5dce3 2024-11-20T11:14:34,708 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(959): stopping server 448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,708 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 349e25c47fdd7d8457f465670572f372 2024-11-20T11:14:34,709 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 07feff3ab7eab5622741a0364ece2464 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a244f5f504c74b0b1dff6963a37aff0 2024-11-20T11:14:34,709 INFO [RS:1;448ea6c54a8c:35099 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;448ea6c54a8c:35099. 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 53a2311878635447fc5975caced0a7d4 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(959): stopping server 448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,709 DEBUG [RS:1;448ea6c54a8c:35099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T11:14:34,709 DEBUG [RS:1;448ea6c54a8c:35099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:34,709 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cb4960297e33c5789bed6704818bf798, disabling compactions & flushes 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;448ea6c54a8c:43507. 2024-11-20T11:14:34,709 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:34,709 DEBUG [RS:0;448ea6c54a8c:43507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T11:14:34,709 DEBUG [RS:0;448ea6c54a8c:43507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:34,709 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:34,709 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-20T11:14:34,709 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. after waiting 0 ms 2024-11-20T11:14:34,709 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1325): Online Regions={031ddda36eb9bb40423f7c9f4322100c=test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c., fb17647ae131913beb1233c85f78d4dc=test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc., 16717e829fb199e0c26298a5116a0b4e=test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e., ec000da7633d9b0b0b955ae7aa7e962f=test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f., ddf1f8c8ecace91d120dbb68cbf18fb1=test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1., 1e9ad74913c92386ef2a8194575ccf6d=test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d., bac309a4ff803986756ba76688d03707=test,eee,1732101267801.bac309a4ff803986756ba76688d03707., 171d75c7056fb7f3c57a7e0ca3724f6d=test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d., 70c2f19bb4361dae17cea153d84e898a=test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a.} 2024-11-20T11:14:34,709 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T11:14:34,709 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T11:14:34,709 DEBUG [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1351): Waiting on 031ddda36eb9bb40423f7c9f4322100c, 073880d856b42f096b9b41dfd0b349af, 16717e829fb199e0c26298a5116a0b4e, 171d75c7056fb7f3c57a7e0ca3724f6d, 1e9ad74913c92386ef2a8194575ccf6d, 70c2f19bb4361dae17cea153d84e898a, bac309a4ff803986756ba76688d03707, ddf1f8c8ecace91d120dbb68cbf18fb1, ec000da7633d9b0b0b955ae7aa7e962f, fb17647ae131913beb1233c85f78d4dc 2024-11-20T11:14:34,710 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-20T11:14:34,710 DEBUG [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(1351): Waiting on 07feff3ab7eab5622741a0364ece2464, 10e6b7dac8e72b3715b09f737ea5dce3, 1588230740, 19c7dd4cb6560dd941c762414bfe9afa, 1a244f5f504c74b0b1dff6963a37aff0, 349e25c47fdd7d8457f465670572f372, 400f5d1df45ccf5378a59a8cd78bb9d8, 4e8a8778f20ffb0d01039b06a807ee6a, 53a2311878635447fc5975caced0a7d4, 6864708d24f9dd54d37dd8e832ca6cb9, 94ab7bd7c2c2041d4db8eb8309fa9cea, ab39ce583e7145069013a6bec41b40c0, cb4960297e33c5789bed6704818bf798, d221d096301219259fde4786f6bb7402 2024-11-20T11:14:34,710 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 031ddda36eb9bb40423f7c9f4322100c, disabling compactions & flushes 2024-11-20T11:14:34,710 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:34,710 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:34,710 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. after waiting 0 ms 2024-11-20T11:14:34,710 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:34,710 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T11:14:34,710 INFO [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T11:14:34,710 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T11:14:34,710 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T11:14:34,710 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T11:14:34,711 INFO [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-20T11:14:34,715 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/031ddda36eb9bb40423f7c9f4322100c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,715 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/cb4960297e33c5789bed6704818bf798/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,718 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:34,718 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:34,718 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cb4960297e33c5789bed6704818bf798: Waiting for close lock at 1732101274708Running coprocessor pre-close hooks at 1732101274709 (+1 ms)Disabling compacts and flushes for region at 1732101274709Disabling writes for close at 1732101274709Writing region close event to WAL at 1732101274710 (+1 ms)Running coprocessor post-close hooks at 1732101274716 (+6 ms)Closed at 1732101274718 (+2 ms) 2024-11-20T11:14:34,718 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 031ddda36eb9bb40423f7c9f4322100c: Waiting for close lock at 1732101274710Running coprocessor pre-close hooks at 1732101274710Disabling compacts and flushes for region at 1732101274710Disabling writes for close at 1732101274710Writing region close event to WAL at 1732101274710Running coprocessor post-close hooks at 1732101274716 (+6 ms)Closed at 1732101274718 (+2 ms) 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732101267801.cb4960297e33c5789bed6704818bf798. 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732101267801.031ddda36eb9bb40423f7c9f4322100c. 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb17647ae131913beb1233c85f78d4dc, disabling compactions & flushes 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d221d096301219259fde4786f6bb7402, disabling compactions & flushes 2024-11-20T11:14:34,719 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:34,719 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732101267801.d221d096301219259fde4786f6bb7402. after waiting 0 ms 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. after waiting 0 ms 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:34,719 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:34,725 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/d221d096301219259fde4786f6bb7402/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,726 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/fb17647ae131913beb1233c85f78d4dc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,726 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:34,726 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d221d096301219259fde4786f6bb7402: Waiting for close lock at 1732101274719Running coprocessor pre-close hooks at 1732101274719Disabling compacts and flushes for region at 1732101274719Disabling writes for close at 1732101274719Writing region close event to WAL at 1732101274720 (+1 ms)Running coprocessor post-close hooks at 1732101274726 (+6 ms)Closed at 1732101274726 2024-11-20T11:14:34,727 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732101267801.d221d096301219259fde4786f6bb7402. 2024-11-20T11:14:34,727 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 94ab7bd7c2c2041d4db8eb8309fa9cea, disabling compactions & flushes 2024-11-20T11:14:34,727 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:34,727 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:34,727 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. after waiting 0 ms 2024-11-20T11:14:34,727 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:34,727 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:34,727 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb17647ae131913beb1233c85f78d4dc: Waiting for close lock at 1732101274719Running coprocessor pre-close hooks at 1732101274719Disabling compacts and flushes for region at 1732101274719Disabling writes for close at 1732101274719Writing region close event to WAL at 1732101274721 (+2 ms)Running coprocessor post-close hooks at 1732101274727 (+6 ms)Closed at 1732101274727 2024-11-20T11:14:34,727 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732101267801.fb17647ae131913beb1233c85f78d4dc. 2024-11-20T11:14:34,727 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 16717e829fb199e0c26298a5116a0b4e, disabling compactions & flushes 2024-11-20T11:14:34,727 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:34,728 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:34,728 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. after waiting 0 ms 2024-11-20T11:14:34,728 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:34,728 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/info/fe2f2131b566400d82ca20f661a09dec is 138, key is test,ccc,1732101267801.0c331e73b55d2ae38913c70b5317c877./info:regioninfo/1732101273824/Put/seqid=0 2024-11-20T11:14:34,733 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/94ab7bd7c2c2041d4db8eb8309fa9cea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,734 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:34,734 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 94ab7bd7c2c2041d4db8eb8309fa9cea: Waiting for close lock at 1732101274727Running coprocessor pre-close hooks at 1732101274727Disabling compacts and flushes for region at 1732101274727Disabling writes for close at 1732101274727Writing region close event to WAL at 1732101274727Running coprocessor post-close hooks at 1732101274734 (+7 ms)Closed at 1732101274734 2024-11-20T11:14:34,734 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732101267801.94ab7bd7c2c2041d4db8eb8309fa9cea. 2024-11-20T11:14:34,734 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 400f5d1df45ccf5378a59a8cd78bb9d8, disabling compactions & flushes 2024-11-20T11:14:34,734 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:34,734 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:34,734 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. after waiting 0 ms 2024-11-20T11:14:34,734 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:34,737 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/16717e829fb199e0c26298a5116a0b4e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,738 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:34,738 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 16717e829fb199e0c26298a5116a0b4e: Waiting for close lock at 1732101274727Running coprocessor pre-close hooks at 1732101274727Disabling compacts and flushes for region at 1732101274727Disabling writes for close at 1732101274728 (+1 ms)Writing region close event to WAL at 1732101274728Running coprocessor post-close hooks at 1732101274738 (+10 ms)Closed at 1732101274738 2024-11-20T11:14:34,738 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732101267801.16717e829fb199e0c26298a5116a0b4e. 2024-11-20T11:14:34,738 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ec000da7633d9b0b0b955ae7aa7e962f, disabling compactions & flushes 2024-11-20T11:14:34,738 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:34,739 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:34,739 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. after waiting 0 ms 2024-11-20T11:14:34,739 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:34,746 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/400f5d1df45ccf5378a59a8cd78bb9d8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,746 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ec000da7633d9b0b0b955ae7aa7e962f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,747 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:34,747 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 400f5d1df45ccf5378a59a8cd78bb9d8: Waiting for close lock at 1732101274734Running coprocessor pre-close hooks at 1732101274734Disabling compacts and flushes for region at 1732101274734Disabling writes for close at 1732101274734Writing region close event to WAL at 1732101274737 (+3 ms)Running coprocessor post-close hooks at 1732101274747 (+10 ms)Closed at 1732101274747 2024-11-20T11:14:34,747 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732101267801.400f5d1df45ccf5378a59a8cd78bb9d8. 2024-11-20T11:14:34,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741869_1045 (size=8195) 2024-11-20T11:14:34,748 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 19c7dd4cb6560dd941c762414bfe9afa, disabling compactions & flushes 2024-11-20T11:14:34,748 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:34,748 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:34,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741869_1045 (size=8195) 2024-11-20T11:14:34,748 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. after waiting 0 ms 2024-11-20T11:14:34,748 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:34,748 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:34,748 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ec000da7633d9b0b0b955ae7aa7e962f: Waiting for close lock at 1732101274738Running coprocessor pre-close hooks at 1732101274738Disabling compacts and flushes for region at 1732101274738Disabling writes for close at 1732101274739 (+1 ms)Writing region close event to WAL at 1732101274739Running coprocessor post-close hooks at 1732101274748 (+9 ms)Closed at 1732101274748 2024-11-20T11:14:34,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741869_1045 (size=8195) 2024-11-20T11:14:34,748 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732101267801.ec000da7633d9b0b0b955ae7aa7e962f. 2024-11-20T11:14:34,749 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ddf1f8c8ecace91d120dbb68cbf18fb1, disabling compactions & flushes 2024-11-20T11:14:34,749 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:34,749 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:34,749 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. after waiting 0 ms 2024-11-20T11:14:34,749 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:34,750 INFO [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/info/fe2f2131b566400d82ca20f661a09dec 2024-11-20T11:14:34,752 INFO [regionserver/448ea6c54a8c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T11:14:34,755 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/19c7dd4cb6560dd941c762414bfe9afa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,756 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:34,756 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 19c7dd4cb6560dd941c762414bfe9afa: Waiting for close lock at 1732101274748Running coprocessor pre-close hooks at 1732101274748Disabling compacts and flushes for region at 1732101274748Disabling writes for close at 1732101274748Writing region close event to WAL at 1732101274749 (+1 ms)Running coprocessor post-close hooks at 1732101274756 (+7 ms)Closed at 1732101274756 2024-11-20T11:14:34,756 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732101267801.19c7dd4cb6560dd941c762414bfe9afa. 2024-11-20T11:14:34,756 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab39ce583e7145069013a6bec41b40c0, disabling compactions & flushes 2024-11-20T11:14:34,757 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:34,757 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:34,757 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. after waiting 0 ms 2024-11-20T11:14:34,757 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:34,760 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ddf1f8c8ecace91d120dbb68cbf18fb1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,761 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/.tmp/info/fe2f2131b566400d82ca20f661a09dec as hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/info/fe2f2131b566400d82ca20f661a09dec 2024-11-20T11:14:34,761 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:34,761 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ddf1f8c8ecace91d120dbb68cbf18fb1: Waiting for close lock at 1732101274749Running coprocessor pre-close hooks at 1732101274749Disabling compacts and flushes for region at 1732101274749Disabling writes for close at 1732101274749Writing region close event to WAL at 1732101274752 (+3 ms)Running coprocessor post-close hooks at 1732101274761 (+9 ms)Closed at 1732101274761 2024-11-20T11:14:34,761 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732101267801.ddf1f8c8ecace91d120dbb68cbf18fb1. 2024-11-20T11:14:34,761 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1e9ad74913c92386ef2a8194575ccf6d, disabling compactions & flushes 2024-11-20T11:14:34,761 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:34,761 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:34,761 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. after waiting 0 ms 2024-11-20T11:14:34,761 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:34,762 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/ab39ce583e7145069013a6bec41b40c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,763 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:34,763 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab39ce583e7145069013a6bec41b40c0: Waiting for close lock at 1732101274756Running coprocessor pre-close hooks at 1732101274756Disabling compacts and flushes for region at 1732101274756Disabling writes for close at 1732101274757 (+1 ms)Writing region close event to WAL at 1732101274757Running coprocessor post-close hooks at 1732101274763 (+6 ms)Closed at 1732101274763 2024-11-20T11:14:34,763 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732101267801.ab39ce583e7145069013a6bec41b40c0. 2024-11-20T11:14:34,764 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4e8a8778f20ffb0d01039b06a807ee6a, disabling compactions & flushes 2024-11-20T11:14:34,764 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:34,764 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:34,764 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. after waiting 0 ms 2024-11-20T11:14:34,764 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:34,766 INFO [regionserver/448ea6c54a8c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T11:14:34,771 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/4e8a8778f20ffb0d01039b06a807ee6a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,771 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1e9ad74913c92386ef2a8194575ccf6d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,772 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:34,772 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:34,772 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4e8a8778f20ffb0d01039b06a807ee6a: Waiting for close lock at 1732101274763Running coprocessor pre-close hooks at 1732101274763Disabling compacts and flushes for region at 1732101274763Disabling writes for close at 1732101274764 (+1 ms)Writing region close event to WAL at 1732101274764Running coprocessor post-close hooks at 1732101274772 (+8 ms)Closed at 1732101274772 2024-11-20T11:14:34,772 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1e9ad74913c92386ef2a8194575ccf6d: Waiting for close lock at 1732101274761Running coprocessor pre-close hooks at 1732101274761Disabling compacts and flushes for region at 1732101274761Disabling writes for close at 1732101274761Writing region close event to WAL at 1732101274762 (+1 ms)Running coprocessor post-close hooks at 1732101274772 (+10 ms)Closed at 1732101274772 2024-11-20T11:14:34,772 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732101267801.1e9ad74913c92386ef2a8194575ccf6d. 2024-11-20T11:14:34,772 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732101267801.4e8a8778f20ffb0d01039b06a807ee6a. 2024-11-20T11:14:34,772 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bac309a4ff803986756ba76688d03707, disabling compactions & flushes 2024-11-20T11:14:34,773 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6864708d24f9dd54d37dd8e832ca6cb9, disabling compactions & flushes 2024-11-20T11:14:34,773 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:34,773 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:34,773 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:34,773 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:34,773 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732101267801.bac309a4ff803986756ba76688d03707. after waiting 0 ms 2024-11-20T11:14:34,773 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:34,773 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. after waiting 0 ms 2024-11-20T11:14:34,773 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:34,773 INFO [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/info/fe2f2131b566400d82ca20f661a09dec, entries=24, sequenceid=71, filesize=8.0 K 2024-11-20T11:14:34,774 INFO [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 64ms, sequenceid=71, compaction requested=false 2024-11-20T11:14:34,788 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/6864708d24f9dd54d37dd8e832ca6cb9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,789 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/bac309a4ff803986756ba76688d03707/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,789 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:34,789 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6864708d24f9dd54d37dd8e832ca6cb9: Waiting for close lock at 1732101274772Running coprocessor pre-close hooks at 1732101274772Disabling compacts and flushes for region at 1732101274772Disabling writes for close at 1732101274773 (+1 ms)Writing region close event to WAL at 1732101274780 (+7 ms)Running coprocessor post-close hooks at 1732101274789 (+9 ms)Closed at 1732101274789 2024-11-20T11:14:34,789 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732101267801.6864708d24f9dd54d37dd8e832ca6cb9. 2024-11-20T11:14:34,790 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bac309a4ff803986756ba76688d03707: Waiting for close lock at 1732101274772Running coprocessor pre-close hooks at 1732101274772Disabling compacts and flushes for region at 1732101274772Disabling writes for close at 1732101274773 (+1 ms)Writing region close event to WAL at 1732101274776 (+3 ms)Running coprocessor post-close hooks at 1732101274789 (+13 ms)Closed at 1732101274789 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 10e6b7dac8e72b3715b09f737ea5dce3, disabling compactions & flushes 2024-11-20T11:14:34,790 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732101267801.bac309a4ff803986756ba76688d03707. 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. after waiting 0 ms 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 171d75c7056fb7f3c57a7e0ca3724f6d, disabling compactions & flushes 2024-11-20T11:14:34,790 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. after waiting 0 ms 2024-11-20T11:14:34,790 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:34,793 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-20T11:14:34,794 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T11:14:34,794 INFO [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T11:14:34,794 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732101274710Running coprocessor pre-close hooks at 1732101274710Disabling compacts and flushes for region at 1732101274710Disabling writes for close at 1732101274710Obtaining lock to block concurrent updates at 1732101274711 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732101274711Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732101274720 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732101274722 (+2 ms)Flushing 1588230740/info: creating writer at 1732101274722Flushing 1588230740/info: appending metadata at 1732101274728 (+6 ms)Flushing 1588230740/info: closing flushed file at 1732101274728Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d22d635: reopening flushed file at 1732101274759 (+31 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 64ms, sequenceid=71, compaction requested=false at 1732101274774 (+15 ms)Writing region close event to WAL at 1732101274789 (+15 ms)Running coprocessor post-close hooks at 1732101274794 (+5 ms)Closed at 1732101274794 2024-11-20T11:14:34,794 DEBUG [RS_CLOSE_META-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T11:14:34,794 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/10e6b7dac8e72b3715b09f737ea5dce3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,795 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:34,795 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 10e6b7dac8e72b3715b09f737ea5dce3: Waiting for close lock at 1732101274790Running coprocessor pre-close hooks at 1732101274790Disabling compacts and flushes for region at 1732101274790Disabling writes for close at 1732101274790Writing region close event to WAL at 1732101274790Running coprocessor post-close hooks at 1732101274795 (+5 ms)Closed at 1732101274795 2024-11-20T11:14:34,795 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732101267801.10e6b7dac8e72b3715b09f737ea5dce3. 2024-11-20T11:14:34,795 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 349e25c47fdd7d8457f465670572f372, disabling compactions & flushes 2024-11-20T11:14:34,795 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:34,795 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:34,795 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732101267801.349e25c47fdd7d8457f465670572f372. after waiting 0 ms 2024-11-20T11:14:34,796 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:34,796 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/171d75c7056fb7f3c57a7e0ca3724f6d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,797 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:34,797 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 171d75c7056fb7f3c57a7e0ca3724f6d: Waiting for close lock at 1732101274790Running coprocessor pre-close hooks at 1732101274790Disabling compacts and flushes for region at 1732101274790Disabling writes for close at 1732101274790Writing region close event to WAL at 1732101274793 (+3 ms)Running coprocessor post-close hooks at 1732101274797 (+4 ms)Closed at 1732101274797 2024-11-20T11:14:34,797 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732101267801.171d75c7056fb7f3c57a7e0ca3724f6d. 2024-11-20T11:14:34,797 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 70c2f19bb4361dae17cea153d84e898a, disabling compactions & flushes 2024-11-20T11:14:34,797 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:34,797 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:34,797 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. after waiting 0 ms 2024-11-20T11:14:34,797 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:34,799 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/349e25c47fdd7d8457f465670572f372/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,800 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:34,800 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 349e25c47fdd7d8457f465670572f372: Waiting for close lock at 1732101274795Running coprocessor pre-close hooks at 1732101274795Disabling compacts and flushes for region at 1732101274795Disabling writes for close at 1732101274795Writing region close event to WAL at 1732101274796 (+1 ms)Running coprocessor post-close hooks at 1732101274800 (+4 ms)Closed at 1732101274800 2024-11-20T11:14:34,800 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732101267801.349e25c47fdd7d8457f465670572f372. 2024-11-20T11:14:34,800 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 07feff3ab7eab5622741a0364ece2464, disabling compactions & flushes 2024-11-20T11:14:34,800 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:34,800 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:34,800 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. after waiting 0 ms 2024-11-20T11:14:34,800 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:34,802 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/70c2f19bb4361dae17cea153d84e898a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,802 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:34,802 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 70c2f19bb4361dae17cea153d84e898a: Waiting for close lock at 1732101274797Running coprocessor pre-close hooks at 1732101274797Disabling compacts and flushes for region at 1732101274797Disabling writes for close at 1732101274797Writing region close event to WAL at 1732101274798 (+1 ms)Running coprocessor post-close hooks at 1732101274802 (+4 ms)Closed at 1732101274802 2024-11-20T11:14:34,803 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732101267801.70c2f19bb4361dae17cea153d84e898a. 2024-11-20T11:14:34,804 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/07feff3ab7eab5622741a0364ece2464/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,805 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:34,805 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 07feff3ab7eab5622741a0364ece2464: Waiting for close lock at 1732101274800Running coprocessor pre-close hooks at 1732101274800Disabling compacts and flushes for region at 1732101274800Disabling writes for close at 1732101274800Writing region close event to WAL at 1732101274801 (+1 ms)Running coprocessor post-close hooks at 1732101274805 (+4 ms)Closed at 1732101274805 2024-11-20T11:14:34,805 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732101267801.07feff3ab7eab5622741a0364ece2464. 2024-11-20T11:14:34,805 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a244f5f504c74b0b1dff6963a37aff0, disabling compactions & flushes 2024-11-20T11:14:34,805 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:34,805 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:34,806 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. after waiting 0 ms 2024-11-20T11:14:34,806 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:34,810 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/1a244f5f504c74b0b1dff6963a37aff0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,810 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:34,810 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a244f5f504c74b0b1dff6963a37aff0: Waiting for close lock at 1732101274805Running coprocessor pre-close hooks at 1732101274805Disabling compacts and flushes for region at 1732101274805Disabling writes for close at 1732101274806 (+1 ms)Writing region close event to WAL at 1732101274806Running coprocessor post-close hooks at 1732101274810 (+4 ms)Closed at 1732101274810 2024-11-20T11:14:34,810 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732101267801.1a244f5f504c74b0b1dff6963a37aff0. 2024-11-20T11:14:34,811 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 53a2311878635447fc5975caced0a7d4, disabling compactions & flushes 2024-11-20T11:14:34,811 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:34,811 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:34,811 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. after waiting 0 ms 2024-11-20T11:14:34,811 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:34,815 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/data/default/test/53a2311878635447fc5975caced0a7d4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T11:14:34,815 INFO [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:34,815 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 53a2311878635447fc5975caced0a7d4: Waiting for close lock at 1732101274811Running coprocessor pre-close hooks at 1732101274811Disabling compacts and flushes for region at 1732101274811Disabling writes for close at 1732101274811Writing region close event to WAL at 1732101274811Running coprocessor post-close hooks at 1732101274815 (+4 ms)Closed at 1732101274815 2024-11-20T11:14:34,816 DEBUG [RS_CLOSE_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732101267801.53a2311878635447fc5975caced0a7d4. 2024-11-20T11:14:34,910 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(976): stopping server 448ea6c54a8c,35099,1732101264437; all regions closed. 2024-11-20T11:14:34,910 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(976): stopping server 448ea6c54a8c,43507,1732101264298; all regions closed. 2024-11-20T11:14:34,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741834_1010 (size=5184) 2024-11-20T11:14:34,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741834_1010 (size=5184) 2024-11-20T11:14:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741865_1041 (size=4204) 2024-11-20T11:14:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741834_1010 (size=5184) 2024-11-20T11:14:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741865_1041 (size=4204) 2024-11-20T11:14:34,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741865_1041 (size=4204) 2024-11-20T11:14:34,918 DEBUG [RS:1;448ea6c54a8c:35099 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs 2024-11-20T11:14:34,918 DEBUG [RS:0;448ea6c54a8c:43507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs 2024-11-20T11:14:34,918 INFO [RS:1;448ea6c54a8c:35099 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 448ea6c54a8c%2C35099%2C1732101264437:(num 1732101266705) 2024-11-20T11:14:34,918 INFO [RS:0;448ea6c54a8c:43507 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 448ea6c54a8c%2C43507%2C1732101264298.meta:.meta(num 1732101273117) 2024-11-20T11:14:34,919 DEBUG [RS:1;448ea6c54a8c:35099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:34,919 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T11:14:34,919 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T11:14:34,919 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.ChoreService(370): Chore service for: regionserver/448ea6c54a8c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T11:14:34,919 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T11:14:34,919 INFO [regionserver/448ea6c54a8c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T11:14:34,919 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T11:14:34,919 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T11:14:34,919 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T11:14:34,920 WARN [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T11:14:34,920 WARN [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af., aborting... java.io.IOException: Failed to report opened region to master: test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T11:14:34,920 ERROR [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server 448ea6c54a8c,35099,1732101264437: Failed to open region test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,jjj,1732101267801.073880d856b42f096b9b41dfd0b349af. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T11:14:34,921 ERROR [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T11:14:34,921 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T11:14:34,922 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T11:14:34,923 INFO [RS:1;448ea6c54a8c:35099 {}] executor.ExecutorService(108): Executor-19-RS_OPEN_REGION-regionserver/448ea6c54a8c:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-20T11:14:34,923 INFO [RS:1;448ea6c54a8c:35099 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35099 2024-11-20T11:14:34,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T11:14:34,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/448ea6c54a8c,35099,1732101264437 2024-11-20T11:14:34,926 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T11:14:34,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741833_1009 (size=7074) 2024-11-20T11:14:34,926 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T11:14:34,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741833_1009 (size=7074) 2024-11-20T11:14:34,926 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [448ea6c54a8c,35099,1732101264437] 2024-11-20T11:14:34,927 DEBUG [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T11:14:34,928 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/448ea6c54a8c,35099,1732101264437 already deleted, retry=false 2024-11-20T11:14:34,929 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 448ea6c54a8c,35099,1732101264437 expired; onlineServers=1 2024-11-20T11:14:34,931 DEBUG [RS:0;448ea6c54a8c:43507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/oldWALs 2024-11-20T11:14:34,931 INFO [RS:0;448ea6c54a8c:43507 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 448ea6c54a8c%2C43507%2C1732101264298:(num 1732101266704) 2024-11-20T11:14:34,931 DEBUG [RS:0;448ea6c54a8c:43507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T11:14:34,931 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T11:14:34,931 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T11:14:34,932 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.ChoreService(370): Chore service for: regionserver/448ea6c54a8c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T11:14:34,932 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T11:14:34,932 INFO [regionserver/448ea6c54a8c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T11:14:34,932 INFO [RS:0;448ea6c54a8c:43507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43507 2024-11-20T11:14:34,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/448ea6c54a8c,43507,1732101264298 2024-11-20T11:14:34,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T11:14:34,935 INFO [RS_OPEN_REGION-regionserver/448ea6c54a8c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 482521752 }, "NonHeapMemoryUsage": { "committed": 163512320, "init": 7667712, "max": -1, "used": 160815352 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "448ea6c54a8c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34784, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 104, "ProcessCallTime_min": 0, "ProcessCallTime_max": 38, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 9, "ProcessCallTime_median": 19, "ProcessCallTime_75th_percentile": 28, "ProcessCallTime_90th_percentile": 34, "ProcessCallTime_95th_percentile": 36, "ProcessCallTime_98th_percentile": 37, "ProcessCallTime_99th_percentile": 37, "ProcessCallTime_99.9th_percentile": 37, "ProcessCallTime_TimeRangeCount_0-1": 104, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 104, "QueueCallTime_min": 0, "QueueCallTime_max": 119, "QueueCallTime_mean": 13, "QueueCallTime_25th_percentile": 29, "QueueCallTime_median": 59, "QueueCallTime_75th_percentile": 89, "QueueCallTime_90th_percentile": 107, "QueueCallTime_95th_percentile": 113, "QueueCallTime_98th_percentile": 116, "QueueCallTime_99th_percentile": 117, "QueueCallTime_99.9th_percentile": 118, "QueueCallTime_TimeRangeCount_0-1": 104, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 104, "TotalCallTime_min": 0, "TotalCallTime_max": 120, "TotalCallTime_mean": 18, "TotalCallTime_25th_percentile": 30, "TotalCallTime_median": 60, "TotalCallTime_75th_percentile": 90, "TotalCallTime_90th_percentile": 108, "TotalCallTime_95th_percentile": 114, "TotalCallTime_98th_percentile": 117, "TotalCallTime_99th_percentile": 118, "TotalCallTime_99.9th_percentile": 119, "TotalCallTime_TimeRangeCount_0-1": 104, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 104, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 657, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 104, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 104, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 303, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 104, "sentBytes": 54766 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "448ea6c54a8c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55317", "tag.serverName": "448ea6c54a8c,35491,1732101264505", "tag.clusterId": "486e3fde-64a7-493d-99c8-877072beb5df", "tag.Context": "regionserver", "tag.Hostname": "448ea6c54a8c", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 28672, "memStoreHeapSize": 51520, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732101264505, "averageRegionSize": 3185, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 74, "totalRowActionRequestCount": 89, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 15, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 15, "ScanTime_min": 0, "ScanTime_max": 13, "ScanTime_mean": 1, "ScanTime_25th_percentile": 3, "ScanTime_median": 6, "ScanTime_75th_percentile": 9, "ScanTime_90th_percentile": 11, "ScanTime_95th_percentile": 12, "ScanTime_98th_percentile": 12, "ScanTime_99th_percentile": 12, "ScanTime_99.9th_percentile": 12, "ScanTime_TimeRangeCount_0-1": 15, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 34, "Put_mean": 4, "Put_25th_percentile": 9, "Put_median": 17, "Put_75th_percentile": 25, "Put_90th_percentile": 30, "Put_95th_percentile": 32, "Put_98th_percentile": 33, "Put_99th_percentile": 33, "Put_99.9th_percentile": 33, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 15, "PutBatch_max": 15, "PutBatch_mean": 15, "PutBatch_25th_percentile": 15, "PutBatch_median": 15, "PutBatch_75th_percentile": 15, "PutBatch_90th_percentile": 15, "PutBatch_95th_percentile": 15, "PutBatch_98th_percentile": 15, "PutBatch_99th_percentile": 15, "PutBatch_99.9th_percentile": 15, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 2, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 15, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4465, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 15, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T11:14:34,935 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T11:14:34,936 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [448ea6c54a8c,43507,1732101264298] 2024-11-20T11:14:34,938 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/448ea6c54a8c,43507,1732101264298 already deleted, retry=false 2024-11-20T11:14:34,938 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 448ea6c54a8c,43507,1732101264298 expired; onlineServers=0 2024-11-20T11:14:34,939 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '448ea6c54a8c,38459,1732101263372' ***** 2024-11-20T11:14:34,939 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T11:14:34,939 INFO [M:0;448ea6c54a8c:38459 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T11:14:34,939 INFO [M:0;448ea6c54a8c:38459 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T11:14:34,940 DEBUG [M:0;448ea6c54a8c:38459 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T11:14:34,940 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T11:14:34,940 DEBUG [M:0;448ea6c54a8c:38459 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T11:14:34,940 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster-HFileCleaner.small.0-1732101266317 {}] cleaner.HFileCleaner(306): Exit Thread[master/448ea6c54a8c:0:becomeActiveMaster-HFileCleaner.small.0-1732101266317,5,FailOnTimeoutGroup] 2024-11-20T11:14:34,940 DEBUG [master/448ea6c54a8c:0:becomeActiveMaster-HFileCleaner.large.0-1732101266310 {}] cleaner.HFileCleaner(306): Exit Thread[master/448ea6c54a8c:0:becomeActiveMaster-HFileCleaner.large.0-1732101266310,5,FailOnTimeoutGroup] 2024-11-20T11:14:34,940 INFO [M:0;448ea6c54a8c:38459 {}] hbase.ChoreService(370): Chore service for: master/448ea6c54a8c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T11:14:34,940 INFO [M:0;448ea6c54a8c:38459 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T11:14:34,940 DEBUG [M:0;448ea6c54a8c:38459 {}] master.HMaster(1795): Stopping service threads 2024-11-20T11:14:34,941 INFO [M:0;448ea6c54a8c:38459 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T11:14:34,941 INFO [M:0;448ea6c54a8c:38459 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T11:14:34,942 INFO [M:0;448ea6c54a8c:38459 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T11:14:34,942 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T11:14:34,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T11:14:34,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T11:14:34,943 DEBUG [M:0;448ea6c54a8c:38459 {}] zookeeper.ZKUtil(347): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T11:14:34,943 WARN [M:0;448ea6c54a8c:38459 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T11:14:34,944 INFO [M:0;448ea6c54a8c:38459 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/.lastflushedseqids 2024-11-20T11:14:34,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741870_1046 (size=1560) 2024-11-20T11:14:34,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741870_1046 (size=1560) 2024-11-20T11:14:34,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741870_1046 (size=1560) 2024-11-20T11:14:34,961 INFO [M:0;448ea6c54a8c:38459 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T11:14:34,961 INFO [M:0;448ea6c54a8c:38459 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T11:14:34,961 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T11:14:34,961 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T11:14:34,961 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T11:14:34,961 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T11:14:34,961 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T11:14:34,961 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.09 KB heapSize=301.48 KB 2024-11-20T11:14:34,978 DEBUG [M:0;448ea6c54a8c:38459 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a19eaf555db14241a1fd779b3b2596e0 is 82, key is hbase:meta,,1/info:regioninfo/1732101273446/Put/seqid=0 2024-11-20T11:14:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741871_1047 (size=6071) 2024-11-20T11:14:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741871_1047 (size=6071) 2024-11-20T11:14:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741871_1047 (size=6071) 2024-11-20T11:14:34,986 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a19eaf555db14241a1fd779b3b2596e0 2024-11-20T11:14:35,008 DEBUG [M:0;448ea6c54a8c:38459 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4a0c55718c3f46a8996c7f0e2cd26ccc is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732101274681/Put/seqid=0 2024-11-20T11:14:35,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741872_1048 (size=12800) 2024-11-20T11:14:35,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741872_1048 (size=12800) 2024-11-20T11:14:35,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741872_1048 (size=12800) 2024-11-20T11:14:35,017 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=246.82 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4a0c55718c3f46a8996c7f0e2cd26ccc 2024-11-20T11:14:35,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T11:14:35,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1013515b6630002, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T11:14:35,029 INFO [RS:1;448ea6c54a8c:35099 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T11:14:35,029 INFO [RS:1;448ea6c54a8c:35099 {}] regionserver.HRegionServer(1031): Exiting; stopping=448ea6c54a8c,35099,1732101264437; zookeeper connection closed. 2024-11-20T11:14:35,029 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d3362c7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d3362c7 2024-11-20T11:14:35,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T11:14:35,038 INFO [RS:0;448ea6c54a8c:43507 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T11:14:35,038 INFO [RS:0;448ea6c54a8c:43507 {}] regionserver.HRegionServer(1031): Exiting; stopping=448ea6c54a8c,43507,1732101264298; zookeeper connection closed. 2024-11-20T11:14:35,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1013515b6630001, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T11:14:35,038 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@8397c26 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@8397c26 2024-11-20T11:14:35,039 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T11:14:35,041 DEBUG [M:0;448ea6c54a8c:38459 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/22eb6084ba114a02b37fec8d54f0ae2d is 69, key is 448ea6c54a8c,35099,1732101264437/rs:state/1732101266360/Put/seqid=0 2024-11-20T11:14:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741873_1049 (size=5447) 2024-11-20T11:14:35,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741873_1049 (size=5447) 2024-11-20T11:14:35,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741873_1049 (size=5447) 2024-11-20T11:14:35,052 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/22eb6084ba114a02b37fec8d54f0ae2d 2024-11-20T11:14:35,058 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 22eb6084ba114a02b37fec8d54f0ae2d 2024-11-20T11:14:35,074 DEBUG [M:0;448ea6c54a8c:38459 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/576916dd251a449ebe1f32719411292d is 52, key is load_balancer_on/state:d/1732101272287/Put/seqid=0 2024-11-20T11:14:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741874_1050 (size=5058) 2024-11-20T11:14:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741874_1050 (size=5058) 2024-11-20T11:14:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741874_1050 (size=5058) 2024-11-20T11:14:35,083 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/576916dd251a449ebe1f32719411292d 2024-11-20T11:14:35,089 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a19eaf555db14241a1fd779b3b2596e0 as hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a19eaf555db14241a1fd779b3b2596e0 2024-11-20T11:14:35,096 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a19eaf555db14241a1fd779b3b2596e0, entries=14, sequenceid=569, filesize=5.9 K 2024-11-20T11:14:35,098 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4a0c55718c3f46a8996c7f0e2cd26ccc as hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4a0c55718c3f46a8996c7f0e2cd26ccc 2024-11-20T11:14:35,104 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4a0c55718c3f46a8996c7f0e2cd26ccc, entries=79, sequenceid=569, filesize=12.5 K 2024-11-20T11:14:35,106 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/22eb6084ba114a02b37fec8d54f0ae2d as hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/22eb6084ba114a02b37fec8d54f0ae2d 2024-11-20T11:14:35,113 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 22eb6084ba114a02b37fec8d54f0ae2d 2024-11-20T11:14:35,114 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/22eb6084ba114a02b37fec8d54f0ae2d, entries=3, sequenceid=569, filesize=5.3 K 2024-11-20T11:14:35,115 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/576916dd251a449ebe1f32719411292d as hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/576916dd251a449ebe1f32719411292d 2024-11-20T11:14:35,122 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/4762ed23-146e-0b19-bc41-8f1316aef34a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/576916dd251a449ebe1f32719411292d, entries=1, sequenceid=569, filesize=4.9 K 2024-11-20T11:14:35,124 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.09 KB/254049, heapSize ~301.41 KB/308648, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=569, compaction requested=false 2024-11-20T11:14:35,125 INFO [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T11:14:35,126 DEBUG [M:0;448ea6c54a8c:38459 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732101274961Disabling compacts and flushes for region at 1732101274961Disabling writes for close at 1732101274961Obtaining lock to block concurrent updates at 1732101274961Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732101274961Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254049, getHeapSize=308648, getOffHeapSize=0, getCellsCount=708 at 1732101274961Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732101274962 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732101274962Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732101274977 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732101274977Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732101274992 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732101275007 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732101275007Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732101275023 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732101275039 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732101275039Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732101275059 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732101275074 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732101275074Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63fde451: reopening flushed file at 1732101275089 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25723f28: reopening flushed file at 1732101275097 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ac7dd7e: reopening flushed file at 1732101275105 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a13c08f: reopening flushed file at 1732101275114 (+9 ms)Finished flush of dataSize ~248.09 KB/254049, heapSize ~301.41 KB/308648, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=569, compaction requested=false at 1732101275124 (+10 ms)Writing region close event to WAL at 1732101275125 (+1 ms)Closed at 1732101275125 2024-11-20T11:14:35,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42077 is added to blk_1073741830_1006 (size=292043) 2024-11-20T11:14:35,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741830_1006 (size=292043) 2024-11-20T11:14:35,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37351 is added to blk_1073741830_1006 (size=292043) 2024-11-20T11:14:35,131 INFO [M:0;448ea6c54a8c:38459 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T11:14:35,131 INFO [M:0;448ea6c54a8c:38459 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38459 2024-11-20T11:14:35,131 INFO [M:0;448ea6c54a8c:38459 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T11:14:35,132 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T11:14:35,235 INFO [M:0;448ea6c54a8c:38459 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T11:14:35,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T11:14:35,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38459-0x1013515b6630000, quorum=127.0.0.1:55317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T11:14:35,240 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@459058f9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T11:14:35,242 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@38c64c15{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T11:14:35,242 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T11:14:35,242 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e0151e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T11:14:35,242 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@60467e4f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir/,STOPPED} 2024-11-20T11:14:35,245 WARN [BP-1883451983-172.17.0.2-1732101259138 heartbeating to localhost/127.0.0.1:35273 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T11:14:35,245 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T11:14:35,245 WARN [BP-1883451983-172.17.0.2-1732101259138 heartbeating to localhost/127.0.0.1:35273 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883451983-172.17.0.2-1732101259138 (Datanode Uuid c1f15936-2099-4e63-8ecb-172c484b2c71) service to localhost/127.0.0.1:35273 2024-11-20T11:14:35,245 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T11:14:35,247 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data5/current/BP-1883451983-172.17.0.2-1732101259138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T11:14:35,247 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data6/current/BP-1883451983-172.17.0.2-1732101259138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T11:14:35,247 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T11:14:35,249 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4caee23f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T11:14:35,250 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@178c0f89{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T11:14:35,250 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T11:14:35,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a49f6d7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T11:14:35,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@414e5e95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir/,STOPPED} 2024-11-20T11:14:35,252 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T11:14:35,252 WARN [BP-1883451983-172.17.0.2-1732101259138 heartbeating to localhost/127.0.0.1:35273 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T11:14:35,252 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T11:14:35,252 WARN [BP-1883451983-172.17.0.2-1732101259138 heartbeating to localhost/127.0.0.1:35273 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883451983-172.17.0.2-1732101259138 (Datanode Uuid d526301a-a1e1-4635-a36e-ab3af0167801) service to localhost/127.0.0.1:35273 2024-11-20T11:14:35,253 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data3/current/BP-1883451983-172.17.0.2-1732101259138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T11:14:35,253 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data4/current/BP-1883451983-172.17.0.2-1732101259138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T11:14:35,253 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T11:14:35,259 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a0ce16d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T11:14:35,259 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1cafe9ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T11:14:35,259 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T11:14:35,259 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62c0eab0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T11:14:35,260 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18a3047c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir/,STOPPED} 2024-11-20T11:14:35,261 WARN [BP-1883451983-172.17.0.2-1732101259138 heartbeating to localhost/127.0.0.1:35273 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T11:14:35,261 WARN [BP-1883451983-172.17.0.2-1732101259138 heartbeating to localhost/127.0.0.1:35273 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883451983-172.17.0.2-1732101259138 (Datanode Uuid 3366fa75-d333-43d9-a106-ab42a7015d7e) service to localhost/127.0.0.1:35273 2024-11-20T11:14:35,262 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data1/current/BP-1883451983-172.17.0.2-1732101259138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T11:14:35,262 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/cluster_7b58a8c0-e63c-8a35-49b6-955ce8d7a8e5/data/data2/current/BP-1883451983-172.17.0.2-1732101259138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T11:14:35,263 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T11:14:35,263 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T11:14:35,263 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T11:14:35,272 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@302cfe51{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T11:14:35,273 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c1eb5a9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T11:14:35,273 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T11:14:35,273 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@378c34a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T11:14:35,273 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@952689a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d220869-45a8-06fb-4243-99b79afc66d7/hadoop.log.dir/,STOPPED} 2024-11-20T11:14:35,284 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T11:14:35,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down