2024-12-05 14:13:22,182 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-05 14:13:22,192 main DEBUG Took 0.008245 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 14:13:22,192 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 14:13:22,192 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 14:13:22,193 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 14:13:22,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,200 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 14:13:22,211 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,212 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,212 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,212 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,213 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,213 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,214 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,214 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,215 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,215 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,216 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,217 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,217 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,218 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,218 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,219 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 14:13:22,219 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,219 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 14:13:22,221 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 14:13:22,222 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 14:13:22,223 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 14:13:22,223 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 14:13:22,224 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 14:13:22,225 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 14:13:22,232 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 14:13:22,234 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 14:13:22,235 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 14:13:22,236 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 14:13:22,236 main DEBUG createAppenders(={Console}) 2024-12-05 14:13:22,236 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-05 14:13:22,237 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-05 14:13:22,237 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-05 14:13:22,237 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 14:13:22,237 main DEBUG OutputStream closed 2024-12-05 14:13:22,238 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 14:13:22,238 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 14:13:22,238 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-05 14:13:22,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 14:13:22,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 14:13:22,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 14:13:22,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 14:13:22,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 14:13:22,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 14:13:22,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 14:13:22,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 14:13:22,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 14:13:22,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 14:13:22,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 14:13:22,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 14:13:22,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 14:13:22,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 14:13:22,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 14:13:22,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 14:13:22,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 14:13:22,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 14:13:22,310 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 14:13:22,310 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-05 14:13:22,311 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 14:13:22,311 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-05T14:13:22,322 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-05 14:13:22,324 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 14:13:22,325 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T14:13:22,518 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf 2024-12-05T14:13:22,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T14:13:22,547 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602, deleteOnExit=true 2024-12-05T14:13:22,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T14:13:22,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/test.cache.data in system properties and HBase conf 2024-12-05T14:13:22,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T14:13:22,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/hadoop.log.dir in system properties and HBase conf 2024-12-05T14:13:22,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T14:13:22,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T14:13:22,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T14:13:22,630 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T14:13:22,711 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T14:13:22,714 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T14:13:22,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T14:13:22,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T14:13:22,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T14:13:22,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T14:13:22,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T14:13:22,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T14:13:22,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T14:13:22,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T14:13:22,718 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/nfs.dump.dir in system properties and HBase conf 2024-12-05T14:13:22,718 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/java.io.tmpdir in system properties and HBase conf 2024-12-05T14:13:22,718 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T14:13:22,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T14:13:22,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T14:13:23,856 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T14:13:23,927 INFO [Time-limited test {}] log.Log(170): Logging initialized @2396ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T14:13:23,993 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T14:13:24,046 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T14:13:24,066 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T14:13:24,067 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T14:13:24,068 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T14:13:24,078 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T14:13:24,081 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/hadoop.log.dir/,AVAILABLE} 2024-12-05T14:13:24,082 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T14:13:24,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/java.io.tmpdir/jetty-localhost-46577-hadoop-hdfs-3_4_1-tests_jar-_-any-9737390619570699531/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T14:13:24,256 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:46577} 2024-12-05T14:13:24,256 INFO [Time-limited test {}] server.Server(415): Started @2726ms 2024-12-05T14:13:24,876 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T14:13:24,882 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T14:13:24,882 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T14:13:24,883 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T14:13:24,883 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T14:13:24,883 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/hadoop.log.dir/,AVAILABLE} 2024-12-05T14:13:24,884 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T14:13:24,976 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/java.io.tmpdir/jetty-localhost-39155-hadoop-hdfs-3_4_1-tests_jar-_-any-15078935426237094974/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T14:13:24,977 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:39155} 2024-12-05T14:13:24,977 INFO [Time-limited test {}] server.Server(415): Started @3447ms 2024-12-05T14:13:25,021 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T14:13:25,116 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T14:13:25,121 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T14:13:25,122 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T14:13:25,123 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T14:13:25,123 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T14:13:25,125 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/hadoop.log.dir/,AVAILABLE} 2024-12-05T14:13:25,125 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T14:13:25,239 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/java.io.tmpdir/jetty-localhost-46617-hadoop-hdfs-3_4_1-tests_jar-_-any-1236785108351842716/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T14:13:25,240 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:46617} 2024-12-05T14:13:25,240 INFO [Time-limited test {}] server.Server(415): Started @3710ms 2024-12-05T14:13:25,243 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T14:13:25,275 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T14:13:25,281 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T14:13:25,283 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T14:13:25,283 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T14:13:25,283 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T14:13:25,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/hadoop.log.dir/,AVAILABLE} 2024-12-05T14:13:25,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T14:13:25,376 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/java.io.tmpdir/jetty-localhost-46571-hadoop-hdfs-3_4_1-tests_jar-_-any-428938806229617490/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T14:13:25,377 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:46571} 2024-12-05T14:13:25,377 INFO [Time-limited test {}] server.Server(415): Started @3847ms 2024-12-05T14:13:25,379 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T14:13:26,773 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/data/data2/current/BP-1652511958-172.17.0.2-1733408003271/current, will proceed with Du for space computation calculation, 2024-12-05T14:13:26,773 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/data/data1/current/BP-1652511958-172.17.0.2-1733408003271/current, will proceed with Du for space computation calculation, 2024-12-05T14:13:26,773 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/data/data3/current/BP-1652511958-172.17.0.2-1733408003271/current, will proceed with Du for space computation calculation, 2024-12-05T14:13:26,773 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/data/data4/current/BP-1652511958-172.17.0.2-1733408003271/current, will proceed with Du for space computation calculation, 2024-12-05T14:13:26,803 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T14:13:26,803 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T14:13:26,844 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/data/data5/current/BP-1652511958-172.17.0.2-1733408003271/current, will proceed with Du for space computation calculation, 2024-12-05T14:13:26,845 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/data/data6/current/BP-1652511958-172.17.0.2-1733408003271/current, will proceed with Du for space computation calculation, 2024-12-05T14:13:26,850 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd7aa836924c63490 with lease ID 0xace638fc5a649075: Processing first storage report for DS-a9dc01fb-9497-4dc5-b5a2-ed0f56e6f176 from datanode DatanodeRegistration(127.0.0.1:38807, datanodeUuid=a0c27af2-2fe0-4079-8c73-d366e53bde6e, infoPort=35641, infoSecurePort=0, ipcPort=46787, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271) 2024-12-05T14:13:26,851 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd7aa836924c63490 with lease ID 0xace638fc5a649075: from storage DS-a9dc01fb-9497-4dc5-b5a2-ed0f56e6f176 node DatanodeRegistration(127.0.0.1:38807, datanodeUuid=a0c27af2-2fe0-4079-8c73-d366e53bde6e, infoPort=35641, infoSecurePort=0, ipcPort=46787, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T14:13:26,851 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x478e4088f27e5da3 with lease ID 0xace638fc5a649074: Processing first storage report for DS-0e75615e-d195-4475-b542-d756761e1516 from datanode DatanodeRegistration(127.0.0.1:44997, datanodeUuid=b66af102-6b59-4b76-b33b-46c514b1eb47, infoPort=35745, infoSecurePort=0, ipcPort=45631, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271) 2024-12-05T14:13:26,851 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x478e4088f27e5da3 with lease ID 0xace638fc5a649074: from storage DS-0e75615e-d195-4475-b542-d756761e1516 node DatanodeRegistration(127.0.0.1:44997, datanodeUuid=b66af102-6b59-4b76-b33b-46c514b1eb47, infoPort=35745, infoSecurePort=0, ipcPort=45631, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T14:13:26,851 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd7aa836924c63490 with lease ID 0xace638fc5a649075: Processing first storage report for DS-287361a6-1a3c-4106-8ee0-5f9ac826fb81 from datanode DatanodeRegistration(127.0.0.1:38807, datanodeUuid=a0c27af2-2fe0-4079-8c73-d366e53bde6e, infoPort=35641, infoSecurePort=0, ipcPort=46787, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271) 2024-12-05T14:13:26,851 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd7aa836924c63490 with lease ID 0xace638fc5a649075: from storage DS-287361a6-1a3c-4106-8ee0-5f9ac826fb81 node DatanodeRegistration(127.0.0.1:38807, datanodeUuid=a0c27af2-2fe0-4079-8c73-d366e53bde6e, infoPort=35641, infoSecurePort=0, ipcPort=46787, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T14:13:26,852 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x478e4088f27e5da3 with lease ID 0xace638fc5a649074: Processing first storage report for DS-d58c7eec-1d20-484f-aa8b-86433c66bbdf from datanode DatanodeRegistration(127.0.0.1:44997, datanodeUuid=b66af102-6b59-4b76-b33b-46c514b1eb47, infoPort=35745, infoSecurePort=0, ipcPort=45631, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271) 2024-12-05T14:13:26,852 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x478e4088f27e5da3 with lease ID 0xace638fc5a649074: from storage DS-d58c7eec-1d20-484f-aa8b-86433c66bbdf node DatanodeRegistration(127.0.0.1:44997, datanodeUuid=b66af102-6b59-4b76-b33b-46c514b1eb47, infoPort=35745, infoSecurePort=0, ipcPort=45631, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T14:13:26,866 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T14:13:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xedef50a31cc90341 with lease ID 0xace638fc5a649076: Processing first storage report for DS-7c4ddf2e-589d-4886-a4d7-a998c1b0e99b from datanode DatanodeRegistration(127.0.0.1:32959, datanodeUuid=8e088d0d-786f-44e8-88f6-90d129f2c890, infoPort=33877, infoSecurePort=0, ipcPort=45351, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271) 2024-12-05T14:13:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xedef50a31cc90341 with lease ID 0xace638fc5a649076: from storage DS-7c4ddf2e-589d-4886-a4d7-a998c1b0e99b node DatanodeRegistration(127.0.0.1:32959, datanodeUuid=8e088d0d-786f-44e8-88f6-90d129f2c890, infoPort=33877, infoSecurePort=0, ipcPort=45351, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T14:13:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xedef50a31cc90341 with lease ID 0xace638fc5a649076: Processing first storage report for DS-74269660-8494-476b-9631-449eeaedd8f0 from datanode DatanodeRegistration(127.0.0.1:32959, datanodeUuid=8e088d0d-786f-44e8-88f6-90d129f2c890, infoPort=33877, infoSecurePort=0, ipcPort=45351, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271) 2024-12-05T14:13:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xedef50a31cc90341 with lease ID 0xace638fc5a649076: from storage DS-74269660-8494-476b-9631-449eeaedd8f0 node DatanodeRegistration(127.0.0.1:32959, datanodeUuid=8e088d0d-786f-44e8-88f6-90d129f2c890, infoPort=33877, infoSecurePort=0, ipcPort=45351, storageInfo=lv=-57;cid=testClusterID;nsid=1792985747;c=1733408003271), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T14:13:26,878 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf 2024-12-05T14:13:26,939 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/zookeeper_0, clientPort=62933, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91dc2fbc-ada9-ba63-a6ea-3390c28033bf/cluster_5343d7cc-25c6-f0ab-3aa1-b4354eb57602/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T14:13:26,948 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62933 2024-12-05T14:13:26,957 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:26,960 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:27,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741825_1001 (size=7) 2024-12-05T14:13:27,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741825_1001 (size=7) 2024-12-05T14:13:27,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741825_1001 (size=7) 2024-12-05T14:13:27,576 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 with version=8 2024-12-05T14:13:27,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/hbase-staging 2024-12-05T14:13:27,642 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T14:13:27,871 INFO [Time-limited test {}] client.ConnectionUtils(128): master/218194e0d8ec:0 server-side Connection retries=45 2024-12-05T14:13:27,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:27,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:27,883 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T14:13:27,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:27,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T14:13:27,997 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T14:13:28,052 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T14:13:28,062 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T14:13:28,067 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T14:13:28,091 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3017 (auto-detected) 2024-12-05T14:13:28,093 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T14:13:28,111 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38561 2024-12-05T14:13:28,135 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38561 connecting to ZooKeeper ensemble=127.0.0.1:62933 2024-12-05T14:13:28,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385610x0, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T14:13:28,315 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38561-0x101a5d71eb70000 connected 2024-12-05T14:13:28,423 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:13:28,446 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41, hbase.cluster.distributed=false 2024-12-05T14:13:28,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T14:13:28,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38561 2024-12-05T14:13:28,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38561 2024-12-05T14:13:28,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38561 2024-12-05T14:13:28,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38561 2024-12-05T14:13:28,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38561 2024-12-05T14:13:28,563 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/218194e0d8ec:0 server-side Connection retries=45 2024-12-05T14:13:28,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,565 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T14:13:28,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T14:13:28,567 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T14:13:28,569 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T14:13:28,570 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40363 2024-12-05T14:13:28,572 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40363 connecting to ZooKeeper ensemble=127.0.0.1:62933 2024-12-05T14:13:28,573 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,576 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403630x0, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T14:13:28,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:403630x0, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:13:28,595 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40363-0x101a5d71eb70001 connected 2024-12-05T14:13:28,599 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T14:13:28,607 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T14:13:28,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T14:13:28,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T14:13:28,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40363 2024-12-05T14:13:28,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40363 2024-12-05T14:13:28,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40363 2024-12-05T14:13:28,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40363 2024-12-05T14:13:28,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40363 2024-12-05T14:13:28,632 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/218194e0d8ec:0 server-side Connection retries=45 2024-12-05T14:13:28,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,633 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T14:13:28,633 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,633 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T14:13:28,633 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T14:13:28,633 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T14:13:28,634 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45449 2024-12-05T14:13:28,635 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45449 connecting to ZooKeeper ensemble=127.0.0.1:62933 2024-12-05T14:13:28,636 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,640 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454490x0, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T14:13:28,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:454490x0, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:13:28,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45449-0x101a5d71eb70002 connected 2024-12-05T14:13:28,658 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T14:13:28,660 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T14:13:28,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T14:13:28,663 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T14:13:28,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45449 2024-12-05T14:13:28,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45449 2024-12-05T14:13:28,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45449 2024-12-05T14:13:28,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45449 2024-12-05T14:13:28,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45449 2024-12-05T14:13:28,680 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/218194e0d8ec:0 server-side Connection retries=45 2024-12-05T14:13:28,680 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,680 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,680 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T14:13:28,680 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T14:13:28,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T14:13:28,681 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T14:13:28,681 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T14:13:28,682 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36017 2024-12-05T14:13:28,683 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36017 connecting to ZooKeeper ensemble=127.0.0.1:62933 2024-12-05T14:13:28,684 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,686 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360170x0, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T14:13:28,700 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360170x0, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:13:28,700 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36017-0x101a5d71eb70003 connected 2024-12-05T14:13:28,700 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T14:13:28,701 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T14:13:28,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T14:13:28,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T14:13:28,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36017 2024-12-05T14:13:28,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36017 2024-12-05T14:13:28,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36017 2024-12-05T14:13:28,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36017 2024-12-05T14:13:28,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36017 2024-12-05T14:13:28,721 DEBUG [M:0;218194e0d8ec:38561 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;218194e0d8ec:38561 2024-12-05T14:13:28,722 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/218194e0d8ec,38561,1733408007726 2024-12-05T14:13:28,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:13:28,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:13:28,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:13:28,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:13:28,739 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/218194e0d8ec,38561,1733408007726 2024-12-05T14:13:28,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T14:13:28,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T14:13:28,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:28,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T14:13:28,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:28,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:28,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:28,769 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T14:13:28,770 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/218194e0d8ec,38561,1733408007726 from backup master directory 2024-12-05T14:13:28,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/218194e0d8ec,38561,1733408007726 2024-12-05T14:13:28,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:13:28,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:13:28,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:13:28,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T14:13:28,784 WARN [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T14:13:28,785 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=218194e0d8ec,38561,1733408007726 2024-12-05T14:13:28,788 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T14:13:28,789 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T14:13:28,840 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/hbase.id] with ID: 39539233-e7ac-4ffc-b25d-6bb3badb36cc 2024-12-05T14:13:28,840 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/.tmp/hbase.id 2024-12-05T14:13:28,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741826_1002 (size=42) 2024-12-05T14:13:28,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741826_1002 (size=42) 2024-12-05T14:13:28,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741826_1002 (size=42) 2024-12-05T14:13:28,854 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/.tmp/hbase.id]:[hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/hbase.id] 2024-12-05T14:13:28,901 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T14:13:28,906 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T14:13:28,924 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-05T14:13:28,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:28,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:28,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:28,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:28,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741827_1003 (size=196) 2024-12-05T14:13:28,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741827_1003 (size=196) 2024-12-05T14:13:28,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741827_1003 (size=196) 2024-12-05T14:13:28,968 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T14:13:28,970 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T14:13:28,983 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T14:13:28,987 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:13:29,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741828_1004 (size=1189) 2024-12-05T14:13:29,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741828_1004 (size=1189) 2024-12-05T14:13:29,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741828_1004 (size=1189) 2024-12-05T14:13:29,035 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/data/master/store 2024-12-05T14:13:29,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741829_1005 (size=34) 2024-12-05T14:13:29,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741829_1005 (size=34) 2024-12-05T14:13:29,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741829_1005 (size=34) 2024-12-05T14:13:29,062 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T14:13:29,065 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:29,067 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T14:13:29,067 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:13:29,068 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:13:29,070 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T14:13:29,070 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:13:29,071 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:13:29,072 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733408009067Disabling compacts and flushes for region at 1733408009067Disabling writes for close at 1733408009070 (+3 ms)Writing region close event to WAL at 1733408009070Closed at 1733408009070 2024-12-05T14:13:29,074 WARN [master/218194e0d8ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/data/master/store/.initializing 2024-12-05T14:13:29,075 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/WALs/218194e0d8ec,38561,1733408007726 2024-12-05T14:13:29,084 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T14:13:29,102 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=218194e0d8ec%2C38561%2C1733408007726, suffix=, logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/WALs/218194e0d8ec,38561,1733408007726, archiveDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/oldWALs, maxLogs=10 2024-12-05T14:13:29,125 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/WALs/218194e0d8ec,38561,1733408007726/218194e0d8ec%2C38561%2C1733408007726.1733408009108, exclude list is [], retry=0 2024-12-05T14:13:29,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44997,DS-0e75615e-d195-4475-b542-d756761e1516,DISK] 2024-12-05T14:13:29,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38807,DS-a9dc01fb-9497-4dc5-b5a2-ed0f56e6f176,DISK] 2024-12-05T14:13:29,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32959,DS-7c4ddf2e-589d-4886-a4d7-a998c1b0e99b,DISK] 2024-12-05T14:13:29,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T14:13:29,178 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/WALs/218194e0d8ec,38561,1733408007726/218194e0d8ec%2C38561%2C1733408007726.1733408009108 2024-12-05T14:13:29,179 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35745:35745),(127.0.0.1/127.0.0.1:35641:35641),(127.0.0.1/127.0.0.1:33877:33877)] 2024-12-05T14:13:29,179 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T14:13:29,180 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:29,182 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,183 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T14:13:29,238 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:29,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T14:13:29,245 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:29,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T14:13:29,249 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:29,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T14:13:29,253 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:29,254 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,257 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,258 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,263 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,264 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,266 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T14:13:29,269 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T14:13:29,273 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:29,274 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64350341, jitterRate=-0.04110519587993622}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T14:13:29,280 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733408009193Initializing all the Stores at 1733408009195 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408009195Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408009196 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408009196Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408009196Cleaning up temporary data from old regions at 1733408009264 (+68 ms)Region opened successfully at 1733408009280 (+16 ms) 2024-12-05T14:13:29,281 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T14:13:29,309 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c80f419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=218194e0d8ec/172.17.0.2:0 2024-12-05T14:13:29,334 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T14:13:29,343 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:29,343 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:29,346 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:29,347 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T14:13:29,351 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T14:13:29,352 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T14:13:29,373 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T14:13:29,380 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T14:13:29,457 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T14:13:29,461 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T14:13:29,464 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T14:13:29,478 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T14:13:29,480 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T14:13:29,484 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T14:13:29,493 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T14:13:29,495 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T14:13:29,504 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T14:13:29,526 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T14:13:29,535 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T14:13:29,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T14:13:29,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T14:13:29,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T14:13:29,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T14:13:29,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,550 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=218194e0d8ec,38561,1733408007726, sessionid=0x101a5d71eb70000, setting cluster-up flag (Was=false) 2024-12-05T14:13:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,615 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T14:13:29,621 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=218194e0d8ec,38561,1733408007726 2024-12-05T14:13:29,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:29,678 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T14:13:29,681 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=218194e0d8ec,38561,1733408007726 2024-12-05T14:13:29,690 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T14:13:29,713 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(746): ClusterId : 39539233-e7ac-4ffc-b25d-6bb3badb36cc 2024-12-05T14:13:29,713 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(746): ClusterId : 39539233-e7ac-4ffc-b25d-6bb3badb36cc 2024-12-05T14:13:29,713 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(746): ClusterId : 39539233-e7ac-4ffc-b25d-6bb3badb36cc 2024-12-05T14:13:29,715 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T14:13:29,715 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T14:13:29,715 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T14:13:29,727 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T14:13:29,727 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T14:13:29,727 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T14:13:29,728 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T14:13:29,728 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T14:13:29,728 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T14:13:29,742 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T14:13:29,742 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T14:13:29,742 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T14:13:29,743 DEBUG [RS:0;218194e0d8ec:40363 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a0c261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=218194e0d8ec/172.17.0.2:0 2024-12-05T14:13:29,743 DEBUG [RS:1;218194e0d8ec:45449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c726dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=218194e0d8ec/172.17.0.2:0 2024-12-05T14:13:29,743 DEBUG [RS:2;218194e0d8ec:36017 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dda024a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=218194e0d8ec/172.17.0.2:0 2024-12-05T14:13:29,755 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:29,756 DEBUG [RS:0;218194e0d8ec:40363 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;218194e0d8ec:40363 2024-12-05T14:13:29,758 DEBUG [RS:1;218194e0d8ec:45449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;218194e0d8ec:45449 2024-12-05T14:13:29,759 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T14:13:29,759 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T14:13:29,759 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T14:13:29,759 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T14:13:29,759 DEBUG [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T14:13:29,759 DEBUG [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T14:13:29,760 DEBUG [RS:2;218194e0d8ec:36017 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;218194e0d8ec:36017 2024-12-05T14:13:29,760 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T14:13:29,760 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T14:13:29,760 DEBUG [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T14:13:29,762 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(2659): reportForDuty to master=218194e0d8ec,38561,1733408007726 with port=36017, startcode=1733408008679 2024-12-05T14:13:29,762 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(2659): reportForDuty to master=218194e0d8ec,38561,1733408007726 with port=40363, startcode=1733408008532 2024-12-05T14:13:29,762 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(2659): reportForDuty to master=218194e0d8ec,38561,1733408007726 with port=45449, startcode=1733408008631 2024-12-05T14:13:29,764 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T14:13:29,770 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T14:13:29,773 DEBUG [RS:0;218194e0d8ec:40363 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T14:13:29,773 DEBUG [RS:1;218194e0d8ec:45449 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T14:13:29,773 DEBUG [RS:2;218194e0d8ec:36017 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T14:13:29,776 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 218194e0d8ec,38561,1733408007726 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T14:13:29,782 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/218194e0d8ec:0, corePoolSize=5, maxPoolSize=5 2024-12-05T14:13:29,783 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/218194e0d8ec:0, corePoolSize=5, maxPoolSize=5 2024-12-05T14:13:29,783 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/218194e0d8ec:0, corePoolSize=5, maxPoolSize=5 2024-12-05T14:13:29,783 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/218194e0d8ec:0, corePoolSize=5, maxPoolSize=5 2024-12-05T14:13:29,783 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/218194e0d8ec:0, corePoolSize=10, maxPoolSize=10 2024-12-05T14:13:29,783 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,783 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/218194e0d8ec:0, corePoolSize=2, maxPoolSize=2 2024-12-05T14:13:29,784 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,785 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408039784 2024-12-05T14:13:29,786 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T14:13:29,787 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T14:13:29,791 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T14:13:29,791 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T14:13:29,791 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T14:13:29,791 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T14:13:29,794 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,798 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T14:13:29,799 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T14:13:29,800 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T14:13:29,801 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:29,802 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T14:13:29,804 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T14:13:29,805 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T14:13:29,806 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50761, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T14:13:29,806 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47311, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T14:13:29,806 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41273, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T14:13:29,807 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/218194e0d8ec:0:becomeActiveMaster-HFileCleaner.large.0-1733408009806,5,FailOnTimeoutGroup] 2024-12-05T14:13:29,807 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/218194e0d8ec:0:becomeActiveMaster-HFileCleaner.small.0-1733408009807,5,FailOnTimeoutGroup] 2024-12-05T14:13:29,807 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,808 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T14:13:29,809 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,809 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,809 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,809 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T14:13:29,812 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 218194e0d8ec,45449,1733408008631 2024-12-05T14:13:29,815 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] master.ServerManager(517): Registering regionserver=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:29,841 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 218194e0d8ec,36017,1733408008679 2024-12-05T14:13:29,841 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] master.ServerManager(517): Registering regionserver=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:29,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741831_1007 (size=1321) 2024-12-05T14:13:29,845 DEBUG [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:29,845 DEBUG [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42699 2024-12-05T14:13:29,845 DEBUG [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T14:13:29,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741831_1007 (size=1321) 2024-12-05T14:13:29,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741831_1007 (size=1321) 2024-12-05T14:13:29,848 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 218194e0d8ec,40363,1733408008532 2024-12-05T14:13:29,848 DEBUG [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:29,848 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] master.ServerManager(517): Registering regionserver=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:29,848 DEBUG [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42699 2024-12-05T14:13:29,848 DEBUG [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T14:13:29,849 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T14:13:29,850 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:29,852 DEBUG [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:29,852 DEBUG [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42699 2024-12-05T14:13:29,852 DEBUG [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T14:13:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741832_1008 (size=32) 2024-12-05T14:13:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741832_1008 (size=32) 2024-12-05T14:13:29,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741832_1008 (size=32) 2024-12-05T14:13:29,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:29,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T14:13:29,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T14:13:29,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:29,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T14:13:29,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T14:13:29,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:29,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T14:13:29,876 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T14:13:29,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:29,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T14:13:29,879 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T14:13:29,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:29,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:29,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T14:13:29,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740 2024-12-05T14:13:29,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740 2024-12-05T14:13:29,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T14:13:29,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T14:13:29,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T14:13:29,885 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T14:13:29,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T14:13:29,891 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:29,892 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67900465, jitterRate=0.011795774102210999}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T14:13:29,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733408009865Initializing all the Stores at 1733408009866 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408009866Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408009866Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408009866Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408009866Cleaning up temporary data from old regions at 1733408009885 (+19 ms)Region opened successfully at 1733408009894 (+9 ms) 2024-12-05T14:13:29,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T14:13:29,894 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T14:13:29,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T14:13:29,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T14:13:29,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T14:13:29,896 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T14:13:29,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733408009894Disabling compacts and flushes for region at 1733408009894Disabling writes for close at 1733408009894Writing region close event to WAL at 1733408009895 (+1 ms)Closed at 1733408009896 (+1 ms) 2024-12-05T14:13:29,898 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:29,898 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T14:13:29,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T14:13:29,904 DEBUG [RS:2;218194e0d8ec:36017 {}] zookeeper.ZKUtil(111): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/218194e0d8ec,36017,1733408008679 2024-12-05T14:13:29,904 DEBUG [RS:1;218194e0d8ec:45449 {}] zookeeper.ZKUtil(111): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/218194e0d8ec,45449,1733408008631 2024-12-05T14:13:29,904 WARN [RS:2;218194e0d8ec:36017 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T14:13:29,904 WARN [RS:1;218194e0d8ec:45449 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T14:13:29,904 INFO [RS:2;218194e0d8ec:36017 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:13:29,905 INFO [RS:1;218194e0d8ec:45449 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:13:29,905 DEBUG [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,36017,1733408008679 2024-12-05T14:13:29,905 DEBUG [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,45449,1733408008631 2024-12-05T14:13:29,905 DEBUG [RS:0;218194e0d8ec:40363 {}] zookeeper.ZKUtil(111): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/218194e0d8ec,40363,1733408008532 2024-12-05T14:13:29,905 WARN [RS:0;218194e0d8ec:40363 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T14:13:29,905 INFO [RS:0;218194e0d8ec:40363 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:13:29,905 DEBUG [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532 2024-12-05T14:13:29,906 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [218194e0d8ec,45449,1733408008631] 2024-12-05T14:13:29,906 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [218194e0d8ec,36017,1733408008679] 2024-12-05T14:13:29,906 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [218194e0d8ec,40363,1733408008532] 2024-12-05T14:13:29,917 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:29,920 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T14:13:29,932 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T14:13:29,932 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T14:13:29,932 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T14:13:29,943 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T14:13:29,943 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T14:13:29,944 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T14:13:29,948 INFO [RS:0;218194e0d8ec:40363 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T14:13:29,948 INFO [RS:2;218194e0d8ec:36017 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T14:13:29,949 INFO [RS:1;218194e0d8ec:45449 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T14:13:29,949 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,949 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,949 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,950 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T14:13:29,951 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T14:13:29,951 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T14:13:29,955 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T14:13:29,955 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T14:13:29,956 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T14:13:29,956 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,956 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,956 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,957 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0, corePoolSize=2, maxPoolSize=2 2024-12-05T14:13:29,957 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,957 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0, corePoolSize=2, maxPoolSize=2 2024-12-05T14:13:29,958 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0, corePoolSize=2, maxPoolSize=2 2024-12-05T14:13:29,958 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/218194e0d8ec:0, corePoolSize=3, maxPoolSize=3 2024-12-05T14:13:29,958 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:1;218194e0d8ec:45449 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/218194e0d8ec:0, corePoolSize=3, maxPoolSize=3 2024-12-05T14:13:29,958 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,958 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/218194e0d8ec:0, corePoolSize=1, maxPoolSize=1 2024-12-05T14:13:29,959 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/218194e0d8ec:0, corePoolSize=3, maxPoolSize=3 2024-12-05T14:13:29,959 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/218194e0d8ec:0, corePoolSize=3, maxPoolSize=3 2024-12-05T14:13:29,959 DEBUG [RS:0;218194e0d8ec:40363 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/218194e0d8ec:0, corePoolSize=3, maxPoolSize=3 2024-12-05T14:13:29,959 DEBUG [RS:2;218194e0d8ec:36017 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/218194e0d8ec:0, corePoolSize=3, maxPoolSize=3 2024-12-05T14:13:29,960 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,960 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,960 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,960 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,960 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,960 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,961 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,45449,1733408008631-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T14:13:29,961 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,961 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,961 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,961 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,961 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,40363,1733408008532-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T14:13:29,961 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,962 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,962 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,962 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,962 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,962 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,36017,1733408008679-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T14:13:29,983 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T14:13:29,983 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T14:13:29,983 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T14:13:29,985 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,45449,1733408008631-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,985 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,36017,1733408008679-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,985 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,40363,1733408008532-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,986 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,986 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,986 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:29,986 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.Replication(171): 218194e0d8ec,45449,1733408008631 started 2024-12-05T14:13:29,986 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.Replication(171): 218194e0d8ec,36017,1733408008679 started 2024-12-05T14:13:29,986 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.Replication(171): 218194e0d8ec,40363,1733408008532 started 2024-12-05T14:13:30,010 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,010 INFO [RS:0;218194e0d8ec:40363 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,010 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,011 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(1482): Serving as 218194e0d8ec,45449,1733408008631, RpcServer on 218194e0d8ec/172.17.0.2:45449, sessionid=0x101a5d71eb70002 2024-12-05T14:13:30,011 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(1482): Serving as 218194e0d8ec,36017,1733408008679, RpcServer on 218194e0d8ec/172.17.0.2:36017, sessionid=0x101a5d71eb70003 2024-12-05T14:13:30,011 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(1482): Serving as 218194e0d8ec,40363,1733408008532, RpcServer on 218194e0d8ec/172.17.0.2:40363, sessionid=0x101a5d71eb70001 2024-12-05T14:13:30,012 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T14:13:30,012 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T14:13:30,012 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T14:13:30,012 DEBUG [RS:1;218194e0d8ec:45449 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 218194e0d8ec,45449,1733408008631 2024-12-05T14:13:30,012 DEBUG [RS:0;218194e0d8ec:40363 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 218194e0d8ec,40363,1733408008532 2024-12-05T14:13:30,012 DEBUG [RS:2;218194e0d8ec:36017 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 218194e0d8ec,36017,1733408008679 2024-12-05T14:13:30,012 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '218194e0d8ec,40363,1733408008532' 2024-12-05T14:13:30,012 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '218194e0d8ec,36017,1733408008679' 2024-12-05T14:13:30,012 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '218194e0d8ec,45449,1733408008631' 2024-12-05T14:13:30,012 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T14:13:30,012 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T14:13:30,012 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T14:13:30,013 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T14:13:30,013 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T14:13:30,013 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T14:13:30,014 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T14:13:30,014 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T14:13:30,014 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T14:13:30,014 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T14:13:30,014 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T14:13:30,014 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T14:13:30,014 DEBUG [RS:0;218194e0d8ec:40363 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 218194e0d8ec,40363,1733408008532 2024-12-05T14:13:30,014 DEBUG [RS:1;218194e0d8ec:45449 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 218194e0d8ec,45449,1733408008631 2024-12-05T14:13:30,014 DEBUG [RS:2;218194e0d8ec:36017 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 218194e0d8ec,36017,1733408008679 2024-12-05T14:13:30,014 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '218194e0d8ec,36017,1733408008679' 2024-12-05T14:13:30,014 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '218194e0d8ec,45449,1733408008631' 2024-12-05T14:13:30,014 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '218194e0d8ec,40363,1733408008532' 2024-12-05T14:13:30,014 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T14:13:30,014 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T14:13:30,014 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T14:13:30,015 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T14:13:30,015 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T14:13:30,015 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T14:13:30,015 DEBUG [RS:1;218194e0d8ec:45449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T14:13:30,015 DEBUG [RS:2;218194e0d8ec:36017 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T14:13:30,015 DEBUG [RS:0;218194e0d8ec:40363 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T14:13:30,015 INFO [RS:1;218194e0d8ec:45449 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T14:13:30,015 INFO [RS:2;218194e0d8ec:36017 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T14:13:30,015 INFO [RS:0;218194e0d8ec:40363 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T14:13:30,015 INFO [RS:2;218194e0d8ec:36017 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T14:13:30,015 INFO [RS:1;218194e0d8ec:45449 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T14:13:30,016 INFO [RS:0;218194e0d8ec:40363 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T14:13:30,071 WARN [218194e0d8ec:38561 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T14:13:30,124 INFO [RS:0;218194e0d8ec:40363 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T14:13:30,124 INFO [RS:2;218194e0d8ec:36017 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T14:13:30,124 INFO [RS:1;218194e0d8ec:45449 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T14:13:30,128 INFO [RS:0;218194e0d8ec:40363 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=218194e0d8ec%2C40363%2C1733408008532, suffix=, logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532, archiveDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/oldWALs, maxLogs=32 2024-12-05T14:13:30,128 INFO [RS:1;218194e0d8ec:45449 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=218194e0d8ec%2C45449%2C1733408008631, suffix=, logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,45449,1733408008631, archiveDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/oldWALs, maxLogs=32 2024-12-05T14:13:30,128 INFO [RS:2;218194e0d8ec:36017 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=218194e0d8ec%2C36017%2C1733408008679, suffix=, logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,36017,1733408008679, archiveDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/oldWALs, maxLogs=32 2024-12-05T14:13:30,143 DEBUG [RS:1;218194e0d8ec:45449 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,45449,1733408008631/218194e0d8ec%2C45449%2C1733408008631.1733408010132, exclude list is [], retry=0 2024-12-05T14:13:30,144 DEBUG [RS:2;218194e0d8ec:36017 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,36017,1733408008679/218194e0d8ec%2C36017%2C1733408008679.1733408010132, exclude list is [], retry=0 2024-12-05T14:13:30,146 DEBUG [RS:0;218194e0d8ec:40363 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532/218194e0d8ec%2C40363%2C1733408008532.1733408010132, exclude list is [], retry=0 2024-12-05T14:13:30,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44997,DS-0e75615e-d195-4475-b542-d756761e1516,DISK] 2024-12-05T14:13:30,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32959,DS-7c4ddf2e-589d-4886-a4d7-a998c1b0e99b,DISK] 2024-12-05T14:13:30,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38807,DS-a9dc01fb-9497-4dc5-b5a2-ed0f56e6f176,DISK] 2024-12-05T14:13:30,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38807,DS-a9dc01fb-9497-4dc5-b5a2-ed0f56e6f176,DISK] 2024-12-05T14:13:30,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44997,DS-0e75615e-d195-4475-b542-d756761e1516,DISK] 2024-12-05T14:13:30,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32959,DS-7c4ddf2e-589d-4886-a4d7-a998c1b0e99b,DISK] 2024-12-05T14:13:30,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44997,DS-0e75615e-d195-4475-b542-d756761e1516,DISK] 2024-12-05T14:13:30,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38807,DS-a9dc01fb-9497-4dc5-b5a2-ed0f56e6f176,DISK] 2024-12-05T14:13:30,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32959,DS-7c4ddf2e-589d-4886-a4d7-a998c1b0e99b,DISK] 2024-12-05T14:13:30,163 INFO [RS:2;218194e0d8ec:36017 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,36017,1733408008679/218194e0d8ec%2C36017%2C1733408008679.1733408010132 2024-12-05T14:13:30,168 DEBUG [RS:2;218194e0d8ec:36017 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35641:35641),(127.0.0.1/127.0.0.1:35745:35745),(127.0.0.1/127.0.0.1:33877:33877)] 2024-12-05T14:13:30,170 INFO [RS:0;218194e0d8ec:40363 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532/218194e0d8ec%2C40363%2C1733408008532.1733408010132 2024-12-05T14:13:30,170 DEBUG [RS:0;218194e0d8ec:40363 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35745:35745),(127.0.0.1/127.0.0.1:35641:35641),(127.0.0.1/127.0.0.1:33877:33877)] 2024-12-05T14:13:30,172 INFO [RS:1;218194e0d8ec:45449 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,45449,1733408008631/218194e0d8ec%2C45449%2C1733408008631.1733408010132 2024-12-05T14:13:30,172 DEBUG [RS:1;218194e0d8ec:45449 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35745:35745),(127.0.0.1/127.0.0.1:33877:33877),(127.0.0.1/127.0.0.1:35641:35641)] 2024-12-05T14:13:30,323 DEBUG [218194e0d8ec:38561 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-05T14:13:30,333 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(204): Hosts are {218194e0d8ec=0} racks are {/default-rack=0} 2024-12-05T14:13:30,338 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T14:13:30,338 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T14:13:30,338 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-05T14:13:30,339 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T14:13:30,339 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T14:13:30,339 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-05T14:13:30,339 INFO [218194e0d8ec:38561 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T14:13:30,339 INFO [218194e0d8ec:38561 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T14:13:30,339 INFO [218194e0d8ec:38561 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-05T14:13:30,339 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T14:13:30,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:30,351 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,40363,1733408008532, state=OPENING 2024-12-05T14:13:30,415 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T14:13:30,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:30,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:30,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:30,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:30,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:30,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:30,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:30,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:30,431 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:30,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:30,613 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T14:13:30,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44405, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T14:13:30,628 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T14:13:30,628 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:13:30,628 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T14:13:30,631 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=218194e0d8ec%2C40363%2C1733408008532.meta, suffix=.meta, logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532, archiveDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/oldWALs, maxLogs=32 2024-12-05T14:13:30,646 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, exclude list is [], retry=0 2024-12-05T14:13:30,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32959,DS-7c4ddf2e-589d-4886-a4d7-a998c1b0e99b,DISK] 2024-12-05T14:13:30,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44997,DS-0e75615e-d195-4475-b542-d756761e1516,DISK] 2024-12-05T14:13:30,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38807,DS-a9dc01fb-9497-4dc5-b5a2-ed0f56e6f176,DISK] 2024-12-05T14:13:30,652 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta 2024-12-05T14:13:30,653 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33877:33877),(127.0.0.1/127.0.0.1:35745:35745),(127.0.0.1/127.0.0.1:35641:35641)] 2024-12-05T14:13:30,653 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T14:13:30,655 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T14:13:30,657 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T14:13:30,661 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T14:13:30,665 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T14:13:30,666 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:30,666 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T14:13:30,667 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T14:13:30,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T14:13:30,671 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T14:13:30,672 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:30,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:30,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T14:13:30,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T14:13:30,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:30,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:30,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T14:13:30,677 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T14:13:30,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:30,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:30,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T14:13:30,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T14:13:30,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:30,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:30,681 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T14:13:30,683 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740 2024-12-05T14:13:30,687 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740 2024-12-05T14:13:30,690 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T14:13:30,690 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T14:13:30,691 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T14:13:30,694 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T14:13:30,696 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61202964, jitterRate=-0.08800476789474487}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T14:13:30,696 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T14:13:30,698 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733408010667Writing region info on filesystem at 1733408010667Initializing all the Stores at 1733408010669 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408010669Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408010669Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408010669Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408010669Cleaning up temporary data from old regions at 1733408010690 (+21 ms)Running coprocessor post-open hooks at 1733408010696 (+6 ms)Region opened successfully at 1733408010698 (+2 ms) 2024-12-05T14:13:30,704 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733408010603 2024-12-05T14:13:30,715 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T14:13:30,716 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T14:13:30,717 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:30,719 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,40363,1733408008532, state=OPEN 2024-12-05T14:13:30,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:30,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:30,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:30,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:30,752 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:30,752 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:30,752 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:30,752 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:30,752 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:30,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T14:13:30,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,40363,1733408008532 in 318 msec 2024-12-05T14:13:30,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T14:13:30,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 857 msec 2024-12-05T14:13:30,767 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:30,767 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T14:13:30,784 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T14:13:30,785 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1] 2024-12-05T14:13:30,804 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T14:13:30,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34677, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T14:13:30,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1100 sec 2024-12-05T14:13:30,828 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733408010828, completionTime=-1 2024-12-05T14:13:30,832 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-05T14:13:30,832 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:30,862 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-05T14:13:30,862 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408070862 2024-12-05T14:13:30,862 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408130862 2024-12-05T14:13:30,862 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-05T14:13:30,864 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:30,873 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,38561,1733408007726-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,873 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,38561,1733408007726-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,873 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,38561,1733408007726-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,875 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-218194e0d8ec:38561, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,876 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,876 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,882 DEBUG [master/218194e0d8ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T14:13:30,910 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.125sec 2024-12-05T14:13:30,912 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T14:13:30,913 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T14:13:30,914 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T14:13:30,915 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T14:13:30,915 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T14:13:30,916 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,38561,1733408007726-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T14:13:30,917 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,38561,1733408007726-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T14:13:30,922 DEBUG [master/218194e0d8ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T14:13:30,923 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T14:13:30,923 INFO [master/218194e0d8ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=218194e0d8ec,38561,1733408007726-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T14:13:30,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T14:13:30,930 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T14:13:30,930 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T14:13:30,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 218194e0d8ec,38561,-1 for getting cluster id 2024-12-05T14:13:30,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T14:13:30,945 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39539233-e7ac-4ffc-b25d-6bb3badb36cc' 2024-12-05T14:13:30,947 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T14:13:30,947 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39539233-e7ac-4ffc-b25d-6bb3badb36cc" 2024-12-05T14:13:30,947 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d3dec2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T14:13:30,947 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [218194e0d8ec,38561,-1] 2024-12-05T14:13:30,950 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T14:13:30,952 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:30,952 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T14:13:30,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T14:13:30,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T14:13:30,961 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1] 2024-12-05T14:13:30,961 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T14:13:30,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T14:13:30,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=218194e0d8ec,38561,1733408007726 2024-12-05T14:13:31,003 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T14:13:31,008 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 218194e0d8ec,38561,1733408007726 2024-12-05T14:13:31,011 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28dae60e 2024-12-05T14:13:31,013 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T14:13:31,015 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T14:13:31,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T14:13:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:31,037 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T14:13:31,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-05T14:13:31,040 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,044 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T14:13:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:13:31,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741837_1013 (size=358) 2024-12-05T14:13:31,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741837_1013 (size=358) 2024-12-05T14:13:31,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741837_1013 (size=358) 2024-12-05T14:13:31,067 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88bd874c8e3a659802f7bbb662036a12, NAME => 'test,,1733408011017.88bd874c8e3a659802f7bbb662036a12.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,067 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 27fe33c5bc228b6793a2d277d2d7d9c2, NAME => 'test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,068 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 894d0c86da04798f8fea0640c4aa6d05, NAME => 'test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741839_1015 (size=42) 2024-12-05T14:13:31,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741839_1015 (size=42) 2024-12-05T14:13:31,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741839_1015 (size=42) 2024-12-05T14:13:31,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733408011017.88bd874c8e3a659802f7bbb662036a12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 88bd874c8e3a659802f7bbb662036a12, disabling compactions & flushes 2024-12-05T14:13:31,105 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:31,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:31,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. after waiting 0 ms 2024-12-05T14:13:31,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:31,105 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:31,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88bd874c8e3a659802f7bbb662036a12: Waiting for close lock at 1733408011105Disabling compacts and flushes for region at 1733408011105Disabling writes for close at 1733408011105Writing region close event to WAL at 1733408011105Closed at 1733408011105 2024-12-05T14:13:31,106 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ffe5657e6841580f09ad342b0a1a3ad, NAME => 'test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741838_1014 (size=45) 2024-12-05T14:13:31,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741838_1014 (size=45) 2024-12-05T14:13:31,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741838_1014 (size=45) 2024-12-05T14:13:31,111 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,111 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 894d0c86da04798f8fea0640c4aa6d05, disabling compactions & flushes 2024-12-05T14:13:31,111 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:31,111 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:31,111 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. after waiting 0 ms 2024-12-05T14:13:31,111 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:31,111 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:31,111 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 894d0c86da04798f8fea0640c4aa6d05: Waiting for close lock at 1733408011111Disabling compacts and flushes for region at 1733408011111Disabling writes for close at 1733408011111Writing region close event to WAL at 1733408011111Closed at 1733408011111 2024-12-05T14:13:31,112 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5c703cd272011369700660d18171711b, NAME => 'test,eee,1733408011017.5c703cd272011369700660d18171711b.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741840_1016 (size=45) 2024-12-05T14:13:31,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741840_1016 (size=45) 2024-12-05T14:13:31,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741840_1016 (size=45) 2024-12-05T14:13:31,120 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,120 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 27fe33c5bc228b6793a2d277d2d7d9c2, disabling compactions & flushes 2024-12-05T14:13:31,120 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:31,120 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:31,120 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. after waiting 0 ms 2024-12-05T14:13:31,121 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:31,121 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:31,121 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 27fe33c5bc228b6793a2d277d2d7d9c2: Waiting for close lock at 1733408011120Disabling compacts and flushes for region at 1733408011120Disabling writes for close at 1733408011120Writing region close event to WAL at 1733408011121 (+1 ms)Closed at 1733408011121 2024-12-05T14:13:31,122 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => abd1856616590a0391ff54d53cfbbb53, NAME => 'test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741841_1017 (size=45) 2024-12-05T14:13:31,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741841_1017 (size=45) 2024-12-05T14:13:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741841_1017 (size=45) 2024-12-05T14:13:31,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 0ffe5657e6841580f09ad342b0a1a3ad, disabling compactions & flushes 2024-12-05T14:13:31,157 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:31,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:31,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. after waiting 0 ms 2024-12-05T14:13:31,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:31,157 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:31,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ffe5657e6841580f09ad342b0a1a3ad: Waiting for close lock at 1733408011157Disabling compacts and flushes for region at 1733408011157Disabling writes for close at 1733408011157Writing region close event to WAL at 1733408011157Closed at 1733408011157 2024-12-05T14:13:31,158 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aba2432d09baa11b4c1ea7026329d127, NAME => 'test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741843_1019 (size=45) 2024-12-05T14:13:31,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741843_1019 (size=45) 2024-12-05T14:13:31,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741843_1019 (size=45) 2024-12-05T14:13:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:13:31,160 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing abd1856616590a0391ff54d53cfbbb53, disabling compactions & flushes 2024-12-05T14:13:31,161 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. 2024-12-05T14:13:31,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. 2024-12-05T14:13:31,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. after waiting 0 ms 2024-12-05T14:13:31,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. 2024-12-05T14:13:31,161 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. 2024-12-05T14:13:31,161 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for abd1856616590a0391ff54d53cfbbb53: Waiting for close lock at 1733408011161Disabling compacts and flushes for region at 1733408011161Disabling writes for close at 1733408011161Writing region close event to WAL at 1733408011161Closed at 1733408011161 2024-12-05T14:13:31,162 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 59e732700952e1f67c04f79b391abf8c, NAME => 'test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741842_1018 (size=45) 2024-12-05T14:13:31,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741842_1018 (size=45) 2024-12-05T14:13:31,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741842_1018 (size=45) 2024-12-05T14:13:31,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1733408011017.5c703cd272011369700660d18171711b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5c703cd272011369700660d18171711b, disabling compactions & flushes 2024-12-05T14:13:31,177 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:31,177 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:31,177 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733408011017.5c703cd272011369700660d18171711b. after waiting 0 ms 2024-12-05T14:13:31,177 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:31,177 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:31,177 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5c703cd272011369700660d18171711b: Waiting for close lock at 1733408011176Disabling compacts and flushes for region at 1733408011176Disabling writes for close at 1733408011177 (+1 ms)Writing region close event to WAL at 1733408011177Closed at 1733408011177 2024-12-05T14:13:31,178 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3fe46a05dda7cc73743c5e741892f632, NAME => 'test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741844_1020 (size=45) 2024-12-05T14:13:31,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741844_1020 (size=45) 2024-12-05T14:13:31,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741844_1020 (size=45) 2024-12-05T14:13:31,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741845_1021 (size=45) 2024-12-05T14:13:31,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741845_1021 (size=45) 2024-12-05T14:13:31,203 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,203 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 59e732700952e1f67c04f79b391abf8c, disabling compactions & flushes 2024-12-05T14:13:31,204 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:31,204 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:31,204 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. after waiting 0 ms 2024-12-05T14:13:31,204 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:31,204 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:31,204 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 59e732700952e1f67c04f79b391abf8c: Waiting for close lock at 1733408011203Disabling compacts and flushes for region at 1733408011203Disabling writes for close at 1733408011204 (+1 ms)Writing region close event to WAL at 1733408011204Closed at 1733408011204 2024-12-05T14:13:31,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741845_1021 (size=45) 2024-12-05T14:13:31,205 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b645f799e60d0a296066576eac0b0285, NAME => 'test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,205 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,205 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing aba2432d09baa11b4c1ea7026329d127, disabling compactions & flushes 2024-12-05T14:13:31,205 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:31,205 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:31,205 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. after waiting 0 ms 2024-12-05T14:13:31,205 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:31,205 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:31,205 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for aba2432d09baa11b4c1ea7026329d127: Waiting for close lock at 1733408011205Disabling compacts and flushes for region at 1733408011205Disabling writes for close at 1733408011205Writing region close event to WAL at 1733408011205Closed at 1733408011205 2024-12-05T14:13:31,206 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 811fb81e434a8467b09b0f283ae0207b, NAME => 'test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741846_1022 (size=45) 2024-12-05T14:13:31,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741846_1022 (size=45) 2024-12-05T14:13:31,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741846_1022 (size=45) 2024-12-05T14:13:31,217 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,218 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 3fe46a05dda7cc73743c5e741892f632, disabling compactions & flushes 2024-12-05T14:13:31,218 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:31,218 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:31,218 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. after waiting 0 ms 2024-12-05T14:13:31,218 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:31,218 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:31,218 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3fe46a05dda7cc73743c5e741892f632: Waiting for close lock at 1733408011218Disabling compacts and flushes for region at 1733408011218Disabling writes for close at 1733408011218Writing region close event to WAL at 1733408011218Closed at 1733408011218 2024-12-05T14:13:31,219 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4edea3efe3103a410142665ac1ea3d83, NAME => 'test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741847_1023 (size=45) 2024-12-05T14:13:31,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741847_1023 (size=45) 2024-12-05T14:13:31,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741847_1023 (size=45) 2024-12-05T14:13:31,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b645f799e60d0a296066576eac0b0285, disabling compactions & flushes 2024-12-05T14:13:31,223 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:31,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:31,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. after waiting 0 ms 2024-12-05T14:13:31,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:31,223 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:31,223 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b645f799e60d0a296066576eac0b0285: Waiting for close lock at 1733408011223Disabling compacts and flushes for region at 1733408011223Disabling writes for close at 1733408011223Writing region close event to WAL at 1733408011223Closed at 1733408011223 2024-12-05T14:13:31,224 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ffa1b4ffdfeddf082998369feea2e28b, NAME => 'test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741848_1024 (size=45) 2024-12-05T14:13:31,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741848_1024 (size=45) 2024-12-05T14:13:31,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741848_1024 (size=45) 2024-12-05T14:13:31,229 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,229 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 811fb81e434a8467b09b0f283ae0207b, disabling compactions & flushes 2024-12-05T14:13:31,229 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:31,229 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:31,229 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. after waiting 0 ms 2024-12-05T14:13:31,229 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:31,229 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:31,230 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 811fb81e434a8467b09b0f283ae0207b: Waiting for close lock at 1733408011229Disabling compacts and flushes for region at 1733408011229Disabling writes for close at 1733408011229Writing region close event to WAL at 1733408011229Closed at 1733408011229 2024-12-05T14:13:31,231 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c129ce5c0755b4946bdefc2d36f50149, NAME => 'test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741849_1025 (size=45) 2024-12-05T14:13:31,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741849_1025 (size=45) 2024-12-05T14:13:31,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741849_1025 (size=45) 2024-12-05T14:13:31,240 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,241 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4edea3efe3103a410142665ac1ea3d83, disabling compactions & flushes 2024-12-05T14:13:31,241 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:31,241 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:31,241 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. after waiting 0 ms 2024-12-05T14:13:31,241 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:31,241 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:31,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741850_1026 (size=45) 2024-12-05T14:13:31,241 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4edea3efe3103a410142665ac1ea3d83: Waiting for close lock at 1733408011241Disabling compacts and flushes for region at 1733408011241Disabling writes for close at 1733408011241Writing region close event to WAL at 1733408011241Closed at 1733408011241 2024-12-05T14:13:31,242 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9dd338d2cdec8bcc235bdb212f336d28, NAME => 'test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741850_1026 (size=45) 2024-12-05T14:13:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741850_1026 (size=45) 2024-12-05T14:13:31,245 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,245 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ffa1b4ffdfeddf082998369feea2e28b, disabling compactions & flushes 2024-12-05T14:13:31,245 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:31,245 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:31,245 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. after waiting 0 ms 2024-12-05T14:13:31,245 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:31,245 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:31,246 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ffa1b4ffdfeddf082998369feea2e28b: Waiting for close lock at 1733408011245Disabling compacts and flushes for region at 1733408011245Disabling writes for close at 1733408011245Writing region close event to WAL at 1733408011245Closed at 1733408011245 2024-12-05T14:13:31,246 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ce07d807a24e01c939692ade9b01d97e, NAME => 'test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741851_1027 (size=45) 2024-12-05T14:13:31,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741851_1027 (size=45) 2024-12-05T14:13:31,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741851_1027 (size=45) 2024-12-05T14:13:31,255 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,255 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c129ce5c0755b4946bdefc2d36f50149, disabling compactions & flushes 2024-12-05T14:13:31,255 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:31,255 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:31,255 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. after waiting 0 ms 2024-12-05T14:13:31,255 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:31,255 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:31,255 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c129ce5c0755b4946bdefc2d36f50149: Waiting for close lock at 1733408011255Disabling compacts and flushes for region at 1733408011255Disabling writes for close at 1733408011255Writing region close event to WAL at 1733408011255Closed at 1733408011255 2024-12-05T14:13:31,256 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88e959e3bb59b49f279be7426309e052, NAME => 'test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741853_1029 (size=45) 2024-12-05T14:13:31,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741853_1029 (size=45) 2024-12-05T14:13:31,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741853_1029 (size=45) 2024-12-05T14:13:31,264 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,264 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ce07d807a24e01c939692ade9b01d97e, disabling compactions & flushes 2024-12-05T14:13:31,264 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:31,264 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:31,264 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. after waiting 0 ms 2024-12-05T14:13:31,264 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:31,264 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:31,264 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ce07d807a24e01c939692ade9b01d97e: Waiting for close lock at 1733408011264Disabling compacts and flushes for region at 1733408011264Disabling writes for close at 1733408011264Writing region close event to WAL at 1733408011264Closed at 1733408011264 2024-12-05T14:13:31,265 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 44e3fd15a9de4a8ceefe8e8505a163f2, NAME => 'test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741852_1028 (size=45) 2024-12-05T14:13:31,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741852_1028 (size=45) 2024-12-05T14:13:31,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741852_1028 (size=45) 2024-12-05T14:13:31,272 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,272 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9dd338d2cdec8bcc235bdb212f336d28, disabling compactions & flushes 2024-12-05T14:13:31,272 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:31,272 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:31,272 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. after waiting 0 ms 2024-12-05T14:13:31,272 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:31,272 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:31,272 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9dd338d2cdec8bcc235bdb212f336d28: Waiting for close lock at 1733408011272Disabling compacts and flushes for region at 1733408011272Disabling writes for close at 1733408011272Writing region close event to WAL at 1733408011272Closed at 1733408011272 2024-12-05T14:13:31,273 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 70c6030455907dcfb74337bc850642ff, NAME => 'test,sss,1733408011017.70c6030455907dcfb74337bc850642ff.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741854_1030 (size=45) 2024-12-05T14:13:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741854_1030 (size=45) 2024-12-05T14:13:31,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741854_1030 (size=45) 2024-12-05T14:13:31,281 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,281 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 88e959e3bb59b49f279be7426309e052, disabling compactions & flushes 2024-12-05T14:13:31,281 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. 2024-12-05T14:13:31,281 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. 2024-12-05T14:13:31,281 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. after waiting 0 ms 2024-12-05T14:13:31,281 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. 2024-12-05T14:13:31,281 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. 2024-12-05T14:13:31,281 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88e959e3bb59b49f279be7426309e052: Waiting for close lock at 1733408011281Disabling compacts and flushes for region at 1733408011281Disabling writes for close at 1733408011281Writing region close event to WAL at 1733408011281Closed at 1733408011281 2024-12-05T14:13:31,282 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cd727192dbe84371a93aad39b906dd3b, NAME => 'test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741855_1031 (size=45) 2024-12-05T14:13:31,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741855_1031 (size=45) 2024-12-05T14:13:31,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741855_1031 (size=45) 2024-12-05T14:13:31,288 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,289 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 44e3fd15a9de4a8ceefe8e8505a163f2, disabling compactions & flushes 2024-12-05T14:13:31,289 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:31,289 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:31,289 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. after waiting 0 ms 2024-12-05T14:13:31,289 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:31,289 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:31,289 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 44e3fd15a9de4a8ceefe8e8505a163f2: Waiting for close lock at 1733408011288Disabling compacts and flushes for region at 1733408011288Disabling writes for close at 1733408011289 (+1 ms)Writing region close event to WAL at 1733408011289Closed at 1733408011289 2024-12-05T14:13:31,290 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 564487ebab3bd815a85a50262b275a91, NAME => 'test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741856_1032 (size=45) 2024-12-05T14:13:31,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741856_1032 (size=45) 2024-12-05T14:13:31,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1733408011017.70c6030455907dcfb74337bc850642ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741857_1033 (size=45) 2024-12-05T14:13:31,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 70c6030455907dcfb74337bc850642ff, disabling compactions & flushes 2024-12-05T14:13:31,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741857_1033 (size=45) 2024-12-05T14:13:31,298 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:31,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:31,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741856_1032 (size=45) 2024-12-05T14:13:31,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. after waiting 0 ms 2024-12-05T14:13:31,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:31,298 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:31,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 70c6030455907dcfb74337bc850642ff: Waiting for close lock at 1733408011297Disabling compacts and flushes for region at 1733408011297Disabling writes for close at 1733408011298 (+1 ms)Writing region close event to WAL at 1733408011298Closed at 1733408011298 2024-12-05T14:13:31,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741857_1033 (size=45) 2024-12-05T14:13:31,299 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c14601cce05aed77304a5058eff93ba1, NAME => 'test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,299 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,299 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing cd727192dbe84371a93aad39b906dd3b, disabling compactions & flushes 2024-12-05T14:13:31,299 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:31,299 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:31,300 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. after waiting 0 ms 2024-12-05T14:13:31,300 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:31,300 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:31,300 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for cd727192dbe84371a93aad39b906dd3b: Waiting for close lock at 1733408011299Disabling compacts and flushes for region at 1733408011299Disabling writes for close at 1733408011300 (+1 ms)Writing region close event to WAL at 1733408011300Closed at 1733408011300 2024-12-05T14:13:31,300 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b620ca010e58c2eb2f2309f4a11c2dba, NAME => 'test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741858_1034 (size=45) 2024-12-05T14:13:31,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741858_1034 (size=45) 2024-12-05T14:13:31,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741858_1034 (size=45) 2024-12-05T14:13:31,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 564487ebab3bd815a85a50262b275a91, disabling compactions & flushes 2024-12-05T14:13:31,309 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:31,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:31,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. after waiting 0 ms 2024-12-05T14:13:31,309 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:31,310 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:31,310 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 564487ebab3bd815a85a50262b275a91: Waiting for close lock at 1733408011309Disabling compacts and flushes for region at 1733408011309Disabling writes for close at 1733408011309Writing region close event to WAL at 1733408011309Closed at 1733408011310 (+1 ms) 2024-12-05T14:13:31,310 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 79e2982faca0d2750624de91bb80c08e, NAME => 'test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741860_1036 (size=45) 2024-12-05T14:13:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741860_1036 (size=45) 2024-12-05T14:13:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741860_1036 (size=45) 2024-12-05T14:13:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741859_1035 (size=45) 2024-12-05T14:13:31,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741859_1035 (size=45) 2024-12-05T14:13:31,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741859_1035 (size=45) 2024-12-05T14:13:31,323 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,323 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c14601cce05aed77304a5058eff93ba1, disabling compactions & flushes 2024-12-05T14:13:31,323 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:31,323 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:31,323 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. after waiting 0 ms 2024-12-05T14:13:31,323 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:31,323 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:31,323 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c14601cce05aed77304a5058eff93ba1: Waiting for close lock at 1733408011323Disabling compacts and flushes for region at 1733408011323Disabling writes for close at 1733408011323Writing region close event to WAL at 1733408011323Closed at 1733408011323 2024-12-05T14:13:31,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741861_1037 (size=45) 2024-12-05T14:13:31,324 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 710e86e039ffc8d96f214260888d4849, NAME => 'test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741861_1037 (size=45) 2024-12-05T14:13:31,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741861_1037 (size=45) 2024-12-05T14:13:31,326 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,326 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 79e2982faca0d2750624de91bb80c08e, disabling compactions & flushes 2024-12-05T14:13:31,326 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:31,326 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:31,326 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. after waiting 0 ms 2024-12-05T14:13:31,326 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:31,326 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:31,326 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 79e2982faca0d2750624de91bb80c08e: Waiting for close lock at 1733408011326Disabling compacts and flushes for region at 1733408011326Disabling writes for close at 1733408011326Writing region close event to WAL at 1733408011326Closed at 1733408011326 2024-12-05T14:13:31,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,327 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b620ca010e58c2eb2f2309f4a11c2dba, disabling compactions & flushes 2024-12-05T14:13:31,327 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a24b719bda0e807d3ebf3858def7531b, NAME => 'test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41 2024-12-05T14:13:31,327 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:31,327 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:31,327 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. after waiting 0 ms 2024-12-05T14:13:31,327 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:31,327 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:31,328 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b620ca010e58c2eb2f2309f4a11c2dba: Waiting for close lock at 1733408011327Disabling compacts and flushes for region at 1733408011327Disabling writes for close at 1733408011327Writing region close event to WAL at 1733408011327Closed at 1733408011327 2024-12-05T14:13:31,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741862_1038 (size=45) 2024-12-05T14:13:31,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741862_1038 (size=45) 2024-12-05T14:13:31,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741862_1038 (size=45) 2024-12-05T14:13:31,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 710e86e039ffc8d96f214260888d4849, disabling compactions & flushes 2024-12-05T14:13:31,336 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:31,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:31,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. after waiting 0 ms 2024-12-05T14:13:31,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:31,336 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:31,336 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 710e86e039ffc8d96f214260888d4849: Waiting for close lock at 1733408011336Disabling compacts and flushes for region at 1733408011336Disabling writes for close at 1733408011336Writing region close event to WAL at 1733408011336Closed at 1733408011336 2024-12-05T14:13:31,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741863_1039 (size=42) 2024-12-05T14:13:31,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741863_1039 (size=42) 2024-12-05T14:13:31,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741863_1039 (size=42) 2024-12-05T14:13:31,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a24b719bda0e807d3ebf3858def7531b, disabling compactions & flushes 2024-12-05T14:13:31,345 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:31,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:31,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. after waiting 0 ms 2024-12-05T14:13:31,345 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:31,345 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:31,346 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a24b719bda0e807d3ebf3858def7531b: Waiting for close lock at 1733408011345Disabling compacts and flushes for region at 1733408011345Disabling writes for close at 1733408011345Writing region close event to WAL at 1733408011345Closed at 1733408011345 2024-12-05T14:13:31,348 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T14:13:31,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733408011017.88bd874c8e3a659802f7bbb662036a12.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733408011017.5c703cd272011369700660d18171711b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733408011017.70c6030455907dcfb74337bc850642ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,356 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733408011348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733408011348"}]},"ts":"1733408011348"} 2024-12-05T14:13:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:13:31,393 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-05T14:13:31,395 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T14:13:31,398 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733408011395"}]},"ts":"1733408011395"} 2024-12-05T14:13:31,402 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-05T14:13:31,403 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {218194e0d8ec=0} racks are {/default-rack=0} 2024-12-05T14:13:31,420 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T14:13:31,420 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T14:13:31,420 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-05T14:13:31,420 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T14:13:31,420 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T14:13:31,420 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-05T14:13:31,420 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T14:13:31,420 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T14:13:31,420 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-05T14:13:31,420 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T14:13:31,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88bd874c8e3a659802f7bbb662036a12, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=27fe33c5bc228b6793a2d277d2d7d9c2, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0ffe5657e6841580f09ad342b0a1a3ad, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5c703cd272011369700660d18171711b, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=59e732700952e1f67c04f79b391abf8c, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3fe46a05dda7cc73743c5e741892f632, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b645f799e60d0a296066576eac0b0285, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4edea3efe3103a410142665ac1ea3d83, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ffa1b4ffdfeddf082998369feea2e28b, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9dd338d2cdec8bcc235bdb212f336d28, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ce07d807a24e01c939692ade9b01d97e, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=44e3fd15a9de4a8ceefe8e8505a163f2, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=70c6030455907dcfb74337bc850642ff, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=564487ebab3bd815a85a50262b275a91, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c14601cce05aed77304a5058eff93ba1, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b620ca010e58c2eb2f2309f4a11c2dba, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=710e86e039ffc8d96f214260888d4849, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN}] 2024-12-05T14:13:31,433 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:31,434 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,435 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=710e86e039ffc8d96f214260888d4849, ASSIGN 2024-12-05T14:13:31,437 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=710e86e039ffc8d96f214260888d4849, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,438 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, ASSIGN 2024-12-05T14:13:31,439 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,440 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b620ca010e58c2eb2f2309f4a11c2dba, ASSIGN 2024-12-05T14:13:31,441 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b620ca010e58c2eb2f2309f4a11c2dba, ASSIGN; state=OFFLINE, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:31,443 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c14601cce05aed77304a5058eff93ba1, ASSIGN 2024-12-05T14:13:31,444 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c14601cce05aed77304a5058eff93ba1, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,446 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=564487ebab3bd815a85a50262b275a91, ASSIGN 2024-12-05T14:13:31,448 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=564487ebab3bd815a85a50262b275a91, ASSIGN; state=OFFLINE, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:31,450 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:31,451 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,453 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=70c6030455907dcfb74337bc850642ff, ASSIGN 2024-12-05T14:13:31,455 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=70c6030455907dcfb74337bc850642ff, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,456 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=44e3fd15a9de4a8ceefe8e8505a163f2, ASSIGN 2024-12-05T14:13:31,458 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=44e3fd15a9de4a8ceefe8e8505a163f2, ASSIGN; state=OFFLINE, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:31,459 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:31,461 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,463 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ce07d807a24e01c939692ade9b01d97e, ASSIGN 2024-12-05T14:13:31,464 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ce07d807a24e01c939692ade9b01d97e, ASSIGN; state=OFFLINE, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:31,465 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9dd338d2cdec8bcc235bdb212f336d28, ASSIGN 2024-12-05T14:13:31,467 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9dd338d2cdec8bcc235bdb212f336d28, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,468 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:31,469 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,470 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ffa1b4ffdfeddf082998369feea2e28b, ASSIGN 2024-12-05T14:13:31,472 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ffa1b4ffdfeddf082998369feea2e28b, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,473 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4edea3efe3103a410142665ac1ea3d83, ASSIGN 2024-12-05T14:13:31,474 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4edea3efe3103a410142665ac1ea3d83, ASSIGN; state=OFFLINE, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:31,475 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:31,477 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,478 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b645f799e60d0a296066576eac0b0285, ASSIGN 2024-12-05T14:13:31,480 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b645f799e60d0a296066576eac0b0285, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,481 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3fe46a05dda7cc73743c5e741892f632, ASSIGN 2024-12-05T14:13:31,483 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3fe46a05dda7cc73743c5e741892f632, ASSIGN; state=OFFLINE, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:31,484 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=59e732700952e1f67c04f79b391abf8c, ASSIGN 2024-12-05T14:13:31,485 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=59e732700952e1f67c04f79b391abf8c, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,486 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:31,487 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,488 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:31,489 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,490 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5c703cd272011369700660d18171711b, ASSIGN 2024-12-05T14:13:31,491 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5c703cd272011369700660d18171711b, ASSIGN; state=OFFLINE, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:31,493 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0ffe5657e6841580f09ad342b0a1a3ad, ASSIGN 2024-12-05T14:13:31,494 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0ffe5657e6841580f09ad342b0a1a3ad, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,495 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:31,497 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN; state=OFFLINE, location=218194e0d8ec,40363,1733408008532; forceNewPlan=false, retain=false 2024-12-05T14:13:31,498 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=27fe33c5bc228b6793a2d277d2d7d9c2, ASSIGN 2024-12-05T14:13:31,499 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=27fe33c5bc228b6793a2d277d2d7d9c2, ASSIGN; state=OFFLINE, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:31,501 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88bd874c8e3a659802f7bbb662036a12, ASSIGN 2024-12-05T14:13:31,502 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88bd874c8e3a659802f7bbb662036a12, ASSIGN; state=OFFLINE, location=218194e0d8ec,36017,1733408008679; forceNewPlan=false, retain=false 2024-12-05T14:13:31,588 INFO [218194e0d8ec:38561 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-05T14:13:31,589 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,590 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=564487ebab3bd815a85a50262b275a91, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,591 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,592 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ce07d807a24e01c939692ade9b01d97e, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b620ca010e58c2eb2f2309f4a11c2dba, ASSIGN because future has completed 2024-12-05T14:13:31,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=4edea3efe3103a410142665ac1ea3d83, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,594 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=3fe46a05dda7cc73743c5e741892f632, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure b620ca010e58c2eb2f2309f4a11c2dba, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:31,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=564487ebab3bd815a85a50262b275a91, ASSIGN because future has completed 2024-12-05T14:13:31,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=44e3fd15a9de4a8ceefe8e8505a163f2, ASSIGN because future has completed 2024-12-05T14:13:31,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ce07d807a24e01c939692ade9b01d97e, ASSIGN because future has completed 2024-12-05T14:13:31,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce07d807a24e01c939692ade9b01d97e, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:31,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4edea3efe3103a410142665ac1ea3d83, ASSIGN because future has completed 2024-12-05T14:13:31,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3fe46a05dda7cc73743c5e741892f632, ASSIGN because future has completed 2024-12-05T14:13:31,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fe46a05dda7cc73743c5e741892f632, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:31,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4edea3efe3103a410142665ac1ea3d83, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:31,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44e3fd15a9de4a8ceefe8e8505a163f2, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:31,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 564487ebab3bd815a85a50262b275a91, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:31,632 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5c703cd272011369700660d18171711b, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,633 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,634 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,635 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=79e2982faca0d2750624de91bb80c08e, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,636 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5c703cd272011369700660d18171711b, ASSIGN because future has completed 2024-12-05T14:13:31,636 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=88e959e3bb59b49f279be7426309e052, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c703cd272011369700660d18171711b, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:31,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=27fe33c5bc228b6793a2d277d2d7d9c2, ASSIGN because future has completed 2024-12-05T14:13:31,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN because future has completed 2024-12-05T14:13:31,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, ASSIGN because future has completed 2024-12-05T14:13:31,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN because future has completed 2024-12-05T14:13:31,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN because future has completed 2024-12-05T14:13:31,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27fe33c5bc228b6793a2d277d2d7d9c2, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:31,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,673 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,673 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,674 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN because future has completed 2024-12-05T14:13:31,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=710e86e039ffc8d96f214260888d4849, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN because future has completed 2024-12-05T14:13:31,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN because future has completed 2024-12-05T14:13:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:13:31,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN because future has completed 2024-12-05T14:13:31,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN because future has completed 2024-12-05T14:13:31,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=710e86e039ffc8d96f214260888d4849, ASSIGN because future has completed 2024-12-05T14:13:31,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 710e86e039ffc8d96f214260888d4849, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba2432d09baa11b4c1ea7026329d127, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:31,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c14601cce05aed77304a5058eff93ba1, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=70c6030455907dcfb74337bc850642ff, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=b645f799e60d0a296066576eac0b0285, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=59e732700952e1f67c04f79b391abf8c, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c14601cce05aed77304a5058eff93ba1, ASSIGN because future has completed 2024-12-05T14:13:31,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c14601cce05aed77304a5058eff93ba1, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=70c6030455907dcfb74337bc850642ff, ASSIGN because future has completed 2024-12-05T14:13:31,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9dd338d2cdec8bcc235bdb212f336d28, ASSIGN because future has completed 2024-12-05T14:13:31,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ffa1b4ffdfeddf082998369feea2e28b, ASSIGN because future has completed 2024-12-05T14:13:31,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffa1b4ffdfeddf082998369feea2e28b, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b645f799e60d0a296066576eac0b0285, ASSIGN because future has completed 2024-12-05T14:13:31,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=59e732700952e1f67c04f79b391abf8c, ASSIGN because future has completed 2024-12-05T14:13:31,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59e732700952e1f67c04f79b391abf8c, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure b645f799e60d0a296066576eac0b0285, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dd338d2cdec8bcc235bdb212f336d28, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70c6030455907dcfb74337bc850642ff, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=88bd874c8e3a659802f7bbb662036a12, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0ffe5657e6841580f09ad342b0a1a3ad, ASSIGN because future has completed 2024-12-05T14:13:31,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ffe5657e6841580f09ad342b0a1a3ad, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88bd874c8e3a659802f7bbb662036a12, ASSIGN because future has completed 2024-12-05T14:13:31,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bd874c8e3a659802f7bbb662036a12, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:31,751 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T14:13:31,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51813, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T14:13:31,760 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:31,760 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 44e3fd15a9de4a8ceefe8e8505a163f2, NAME => 'test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-05T14:13:31,761 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,761 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,761 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,761 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,763 INFO [StoreOpener-44e3fd15a9de4a8ceefe8e8505a163f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,765 INFO [StoreOpener-44e3fd15a9de4a8ceefe8e8505a163f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44e3fd15a9de4a8ceefe8e8505a163f2 columnFamilyName family 2024-12-05T14:13:31,765 DEBUG [StoreOpener-44e3fd15a9de4a8ceefe8e8505a163f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,766 INFO [StoreOpener-44e3fd15a9de4a8ceefe8e8505a163f2-1 {}] regionserver.HStore(327): Store=44e3fd15a9de4a8ceefe8e8505a163f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,766 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,767 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,768 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,768 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,768 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,770 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,773 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/44e3fd15a9de4a8ceefe8e8505a163f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,774 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 44e3fd15a9de4a8ceefe8e8505a163f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74115289, jitterRate=0.10440386831760406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,775 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:31,776 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 44e3fd15a9de4a8ceefe8e8505a163f2: Running coprocessor pre-open hook at 1733408011761Writing region info on filesystem at 1733408011761Initializing all the Stores at 1733408011763 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011763Cleaning up temporary data from old regions at 1733408011768 (+5 ms)Running coprocessor post-open hooks at 1733408011775 (+7 ms)Region opened successfully at 1733408011776 (+1 ms) 2024-12-05T14:13:31,779 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2., pid=35, masterSystemTime=1733408011750 2024-12-05T14:13:31,782 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:31,782 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:31,782 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:31,783 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 3fe46a05dda7cc73743c5e741892f632, NAME => 'test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-05T14:13:31,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,783 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,783 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,784 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,784 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44e3fd15a9de4a8ceefe8e8505a163f2, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:31,787 INFO [StoreOpener-3fe46a05dda7cc73743c5e741892f632-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,789 INFO [StoreOpener-3fe46a05dda7cc73743c5e741892f632-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fe46a05dda7cc73743c5e741892f632 columnFamilyName family 2024-12-05T14:13:31,789 DEBUG [StoreOpener-3fe46a05dda7cc73743c5e741892f632-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,790 INFO [StoreOpener-3fe46a05dda7cc73743c5e741892f632-1 {}] regionserver.HStore(327): Store=3fe46a05dda7cc73743c5e741892f632/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,790 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=22 2024-12-05T14:13:31,791 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 44e3fd15a9de4a8ceefe8e8505a163f2, server=218194e0d8ec,45449,1733408008631 in 166 msec 2024-12-05T14:13:31,792 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,793 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,793 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=44e3fd15a9de4a8ceefe8e8505a163f2, ASSIGN in 369 msec 2024-12-05T14:13:31,795 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,798 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/3fe46a05dda7cc73743c5e741892f632/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,798 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 3fe46a05dda7cc73743c5e741892f632; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65375454, jitterRate=-0.025829821825027466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,798 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:31,799 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 3fe46a05dda7cc73743c5e741892f632: Running coprocessor pre-open hook at 1733408011784Writing region info on filesystem at 1733408011784Initializing all the Stores at 1733408011786 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011786Cleaning up temporary data from old regions at 1733408011793 (+7 ms)Running coprocessor post-open hooks at 1733408011799 (+6 ms)Region opened successfully at 1733408011799 2024-12-05T14:13:31,800 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632., pid=33, masterSystemTime=1733408011750 2024-12-05T14:13:31,803 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:31,803 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:31,803 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:31,803 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 564487ebab3bd815a85a50262b275a91, NAME => 'test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-05T14:13:31,804 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,804 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,804 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,804 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,807 INFO [StoreOpener-564487ebab3bd815a85a50262b275a91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,807 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:31,807 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => cd727192dbe84371a93aad39b906dd3b, NAME => 'test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-05T14:13:31,808 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,808 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fe46a05dda7cc73743c5e741892f632, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:31,808 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,808 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,809 INFO [StoreOpener-564487ebab3bd815a85a50262b275a91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 564487ebab3bd815a85a50262b275a91 columnFamilyName family 2024-12-05T14:13:31,809 DEBUG [StoreOpener-564487ebab3bd815a85a50262b275a91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,810 INFO [StoreOpener-564487ebab3bd815a85a50262b275a91-1 {}] regionserver.HStore(327): Store=564487ebab3bd815a85a50262b275a91/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,810 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,810 INFO [StoreOpener-cd727192dbe84371a93aad39b906dd3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,812 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,812 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,813 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,813 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,813 INFO [StoreOpener-cd727192dbe84371a93aad39b906dd3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd727192dbe84371a93aad39b906dd3b columnFamilyName family 2024-12-05T14:13:31,814 DEBUG [StoreOpener-cd727192dbe84371a93aad39b906dd3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,814 INFO [StoreOpener-cd727192dbe84371a93aad39b906dd3b-1 {}] regionserver.HStore(327): Store=cd727192dbe84371a93aad39b906dd3b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,814 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,815 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,816 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,817 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,817 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,817 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,818 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/564487ebab3bd815a85a50262b275a91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,818 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 564487ebab3bd815a85a50262b275a91; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70953493, jitterRate=0.05728943645954132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,818 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:31,818 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 564487ebab3bd815a85a50262b275a91: Running coprocessor pre-open hook at 1733408011804Writing region info on filesystem at 1733408011804Initializing all the Stores at 1733408011806 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011806Cleaning up temporary data from old regions at 1733408011813 (+7 ms)Running coprocessor post-open hooks at 1733408011818 (+5 ms)Region opened successfully at 1733408011818 2024-12-05T14:13:31,819 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,819 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91., pid=36, masterSystemTime=1733408011750 2024-12-05T14:13:31,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-12-05T14:13:31,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 3fe46a05dda7cc73743c5e741892f632, server=218194e0d8ec,45449,1733408008631 in 197 msec 2024-12-05T14:13:31,823 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:31,823 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:31,823 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:31,823 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => b620ca010e58c2eb2f2309f4a11c2dba, NAME => 'test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-05T14:13:31,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3fe46a05dda7cc73743c5e741892f632, ASSIGN in 399 msec 2024-12-05T14:13:31,824 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,824 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,824 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,824 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,825 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/cd727192dbe84371a93aad39b906dd3b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,826 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened cd727192dbe84371a93aad39b906dd3b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61247908, jitterRate=-0.08733505010604858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,826 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:31,826 INFO [StoreOpener-b620ca010e58c2eb2f2309f4a11c2dba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,826 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for cd727192dbe84371a93aad39b906dd3b: Running coprocessor pre-open hook at 1733408011808Writing region info on filesystem at 1733408011808Initializing all the Stores at 1733408011810 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011810Cleaning up temporary data from old regions at 1733408011817 (+7 ms)Running coprocessor post-open hooks at 1733408011826 (+9 ms)Region opened successfully at 1733408011826 2024-12-05T14:13:31,828 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b., pid=40, masterSystemTime=1733408011800 2024-12-05T14:13:31,828 INFO [StoreOpener-b620ca010e58c2eb2f2309f4a11c2dba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b620ca010e58c2eb2f2309f4a11c2dba columnFamilyName family 2024-12-05T14:13:31,828 DEBUG [StoreOpener-b620ca010e58c2eb2f2309f4a11c2dba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,829 INFO [StoreOpener-b620ca010e58c2eb2f2309f4a11c2dba-1 {}] regionserver.HStore(327): Store=b620ca010e58c2eb2f2309f4a11c2dba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,829 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,830 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,830 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,831 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,831 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,833 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,835 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/b620ca010e58c2eb2f2309f4a11c2dba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,836 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened b620ca010e58c2eb2f2309f4a11c2dba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74332341, jitterRate=0.10763819515705109}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,836 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:31,836 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for b620ca010e58c2eb2f2309f4a11c2dba: Running coprocessor pre-open hook at 1733408011824Writing region info on filesystem at 1733408011825 (+1 ms)Initializing all the Stores at 1733408011826 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011826Cleaning up temporary data from old regions at 1733408011831 (+5 ms)Running coprocessor post-open hooks at 1733408011836 (+5 ms)Region opened successfully at 1733408011836 2024-12-05T14:13:31,837 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=564487ebab3bd815a85a50262b275a91, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,837 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:31,837 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:31,837 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. 2024-12-05T14:13:31,837 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => abd1856616590a0391ff54d53cfbbb53, NAME => 'test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-05T14:13:31,838 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,838 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,838 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,838 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,838 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba., pid=31, masterSystemTime=1733408011750 2024-12-05T14:13:31,838 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T14:13:31,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53889, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T14:13:31,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 564487ebab3bd815a85a50262b275a91, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:31,842 INFO [StoreOpener-abd1856616590a0391ff54d53cfbbb53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,845 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=cd727192dbe84371a93aad39b906dd3b, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,846 INFO [StoreOpener-abd1856616590a0391ff54d53cfbbb53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd1856616590a0391ff54d53cfbbb53 columnFamilyName family 2024-12-05T14:13:31,846 DEBUG [StoreOpener-abd1856616590a0391ff54d53cfbbb53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,850 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:31,850 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:31,850 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:31,850 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 5c703cd272011369700660d18171711b, NAME => 'test,eee,1733408011017.5c703cd272011369700660d18171711b.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-05T14:13:31,851 INFO [StoreOpener-abd1856616590a0391ff54d53cfbbb53-1 {}] regionserver.HStore(327): Store=abd1856616590a0391ff54d53cfbbb53/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,851 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,851 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,eee,1733408011017.5c703cd272011369700660d18171711b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,851 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,851 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:31,854 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=564487ebab3bd815a85a50262b275a91. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:31,855 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,857 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,858 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,859 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,859 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,859 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,40363,1733408008532, table=test, region=cd727192dbe84371a93aad39b906dd3b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:31,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure b620ca010e58c2eb2f2309f4a11c2dba, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:31,862 INFO [StoreOpener-5c703cd272011369700660d18171711b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,864 INFO [StoreOpener-5c703cd272011369700660d18171711b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c703cd272011369700660d18171711b columnFamilyName family 2024-12-05T14:13:31,864 DEBUG [StoreOpener-5c703cd272011369700660d18171711b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,865 INFO [StoreOpener-5c703cd272011369700660d18171711b-1 {}] regionserver.HStore(327): Store=5c703cd272011369700660d18171711b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,865 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,865 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=24 2024-12-05T14:13:31,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,40363,1733408008532 in 204 msec 2024-12-05T14:13:31,867 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/5c703cd272011369700660d18171711b 2024-12-05T14:13:31,867 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/5c703cd272011369700660d18171711b 2024-12-05T14:13:31,868 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,869 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,870 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:31,870 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => ffa1b4ffdfeddf082998369feea2e28b, NAME => 'test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-05T14:13:31,871 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,871 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,871 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,871 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN in 443 msec 2024-12-05T14:13:31,872 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/abd1856616590a0391ff54d53cfbbb53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,872 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,873 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened abd1856616590a0391ff54d53cfbbb53; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63551427, jitterRate=-0.053009942173957825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,874 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:31,874 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for abd1856616590a0391ff54d53cfbbb53: Running coprocessor pre-open hook at 1733408011838Writing region info on filesystem at 1733408011838Initializing all the Stores at 1733408011840 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011840Cleaning up temporary data from old regions at 1733408011859 (+19 ms)Running coprocessor post-open hooks at 1733408011874 (+15 ms)Region opened successfully at 1733408011874 2024-12-05T14:13:31,875 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53., pid=46, masterSystemTime=1733408011800 2024-12-05T14:13:31,875 INFO [StoreOpener-ffa1b4ffdfeddf082998369feea2e28b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,878 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/5c703cd272011369700660d18171711b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,878 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. 2024-12-05T14:13:31,878 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. 2024-12-05T14:13:31,879 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:31,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=27 2024-12-05T14:13:31,879 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 5c703cd272011369700660d18171711b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66830361, jitterRate=-0.004150018095970154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,879 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 894d0c86da04798f8fea0640c4aa6d05, NAME => 'test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-05T14:13:31,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure b620ca010e58c2eb2f2309f4a11c2dba, server=218194e0d8ec,45449,1733408008631 in 279 msec 2024-12-05T14:13:31,879 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5c703cd272011369700660d18171711b 2024-12-05T14:13:31,879 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 5c703cd272011369700660d18171711b: Running coprocessor pre-open hook at 1733408011854Writing region info on filesystem at 1733408011854Initializing all the Stores at 1733408011859 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011859Cleaning up temporary data from old regions at 1733408011869 (+10 ms)Running coprocessor post-open hooks at 1733408011879 (+10 ms)Region opened successfully at 1733408011879 2024-12-05T14:13:31,880 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,880 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,880 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,880 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,881 INFO [StoreOpener-ffa1b4ffdfeddf082998369feea2e28b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffa1b4ffdfeddf082998369feea2e28b columnFamilyName family 2024-12-05T14:13:31,881 DEBUG [StoreOpener-ffa1b4ffdfeddf082998369feea2e28b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,882 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733408011017.5c703cd272011369700660d18171711b., pid=37, masterSystemTime=1733408011750 2024-12-05T14:13:31,882 INFO [StoreOpener-ffa1b4ffdfeddf082998369feea2e28b-1 {}] regionserver.HStore(327): Store=ffa1b4ffdfeddf082998369feea2e28b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,883 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,883 INFO [StoreOpener-894d0c86da04798f8fea0640c4aa6d05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b620ca010e58c2eb2f2309f4a11c2dba, ASSIGN in 457 msec 2024-12-05T14:13:31,884 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,885 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,886 INFO [StoreOpener-894d0c86da04798f8fea0640c4aa6d05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 894d0c86da04798f8fea0640c4aa6d05 columnFamilyName family 2024-12-05T14:13:31,886 DEBUG [StoreOpener-894d0c86da04798f8fea0640c4aa6d05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,886 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,886 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,887 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=abd1856616590a0391ff54d53cfbbb53, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,887 INFO [StoreOpener-894d0c86da04798f8fea0640c4aa6d05-1 {}] regionserver.HStore(327): Store=894d0c86da04798f8fea0640c4aa6d05/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,887 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,888 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:31,890 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,888 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:31,891 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,891 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,891 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:31,891 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5c703cd272011369700660d18171711b, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,891 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => ce07d807a24e01c939692ade9b01d97e, NAME => 'test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-05T14:13:31,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:31,892 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,892 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,892 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,892 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,892 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,892 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,896 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c703cd272011369700660d18171711b, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:31,901 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/ffa1b4ffdfeddf082998369feea2e28b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,901 INFO [StoreOpener-ce07d807a24e01c939692ade9b01d97e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,902 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/894d0c86da04798f8fea0640c4aa6d05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,902 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened ffa1b4ffdfeddf082998369feea2e28b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60693327, jitterRate=-0.09559895098209381}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,902 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:31,902 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 894d0c86da04798f8fea0640c4aa6d05; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72739621, jitterRate=0.08390481770038605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,902 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:31,902 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for ffa1b4ffdfeddf082998369feea2e28b: Running coprocessor pre-open hook at 1733408011871Writing region info on filesystem at 1733408011871Initializing all the Stores at 1733408011873 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011873Cleaning up temporary data from old regions at 1733408011886 (+13 ms)Running coprocessor post-open hooks at 1733408011902 (+16 ms)Region opened successfully at 1733408011902 2024-12-05T14:13:31,903 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 894d0c86da04798f8fea0640c4aa6d05: Running coprocessor pre-open hook at 1733408011881Writing region info on filesystem at 1733408011881Initializing all the Stores at 1733408011882 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011882Cleaning up temporary data from old regions at 1733408011892 (+10 ms)Running coprocessor post-open hooks at 1733408011902 (+10 ms)Region opened successfully at 1733408011902 2024-12-05T14:13:31,904 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b., pid=50, masterSystemTime=1733408011838 2024-12-05T14:13:31,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=10 2024-12-05T14:13:31,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,40363,1733408008532 in 206 msec 2024-12-05T14:13:31,904 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05., pid=44, masterSystemTime=1733408011800 2024-12-05T14:13:31,904 INFO [StoreOpener-ce07d807a24e01c939692ade9b01d97e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce07d807a24e01c939692ade9b01d97e columnFamilyName family 2024-12-05T14:13:31,904 DEBUG [StoreOpener-ce07d807a24e01c939692ade9b01d97e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,905 INFO [StoreOpener-ce07d807a24e01c939692ade9b01d97e-1 {}] regionserver.HStore(327): Store=ce07d807a24e01c939692ade9b01d97e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,905 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,907 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,907 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:31,907 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,908 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:31,908 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:31,908 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,908 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN in 482 msec 2024-12-05T14:13:31,909 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 59e732700952e1f67c04f79b391abf8c, NAME => 'test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-05T14:13:31,909 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:31,909 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:31,909 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,909 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,909 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:31,909 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,909 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,909 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => a24b719bda0e807d3ebf3858def7531b, NAME => 'test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-05T14:13:31,910 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,910 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,910 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,910 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,912 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=894d0c86da04798f8fea0640c4aa6d05, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,913 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,913 INFO [StoreOpener-a24b719bda0e807d3ebf3858def7531b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,914 INFO [StoreOpener-59e732700952e1f67c04f79b391abf8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,916 INFO [StoreOpener-a24b719bda0e807d3ebf3858def7531b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a24b719bda0e807d3ebf3858def7531b columnFamilyName family 2024-12-05T14:13:31,916 DEBUG [StoreOpener-a24b719bda0e807d3ebf3858def7531b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:31,917 INFO [StoreOpener-a24b719bda0e807d3ebf3858def7531b-1 {}] regionserver.HStore(327): Store=a24b719bda0e807d3ebf3858def7531b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,917 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,918 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,918 INFO [StoreOpener-59e732700952e1f67c04f79b391abf8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59e732700952e1f67c04f79b391abf8c columnFamilyName family 2024-12-05T14:13:31,918 DEBUG [StoreOpener-59e732700952e1f67c04f79b391abf8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,918 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,919 INFO [StoreOpener-59e732700952e1f67c04f79b391abf8c-1 {}] regionserver.HStore(327): Store=59e732700952e1f67c04f79b391abf8c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,919 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/ce07d807a24e01c939692ade9b01d97e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,920 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,920 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,921 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,921 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,921 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,921 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened ce07d807a24e01c939692ade9b01d97e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66823290, jitterRate=-0.004255384206771851}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,921 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:31,921 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for ce07d807a24e01c939692ade9b01d97e: Running coprocessor pre-open hook at 1733408011892Writing region info on filesystem at 1733408011892Initializing all the Stores at 1733408011896 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011896Cleaning up temporary data from old regions at 1733408011908 (+12 ms)Running coprocessor post-open hooks at 1733408011921 (+13 ms)Region opened successfully at 1733408011921 2024-12-05T14:13:31,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffa1b4ffdfeddf082998369feea2e28b, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:31,923 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,923 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e., pid=32, masterSystemTime=1733408011750 2024-12-05T14:13:31,924 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,924 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,926 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,927 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:31,927 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:31,927 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:31,927 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 4edea3efe3103a410142665ac1ea3d83, NAME => 'test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-05T14:13:31,928 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,928 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,928 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,928 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,930 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,930 INFO [StoreOpener-4edea3efe3103a410142665ac1ea3d83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,931 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/a24b719bda0e807d3ebf3858def7531b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=7 2024-12-05T14:13:31,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,40363,1733408008532 in 243 msec 2024-12-05T14:13:31,932 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened a24b719bda0e807d3ebf3858def7531b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60668475, jitterRate=-0.09596927464008331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,933 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:31,933 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for a24b719bda0e807d3ebf3858def7531b: Running coprocessor pre-open hook at 1733408011910Writing region info on filesystem at 1733408011910Initializing all the Stores at 1733408011912 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011912Cleaning up temporary data from old regions at 1733408011921 (+9 ms)Running coprocessor post-open hooks at 1733408011933 (+12 ms)Region opened successfully at 1733408011933 2024-12-05T14:13:31,934 INFO [StoreOpener-4edea3efe3103a410142665ac1ea3d83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4edea3efe3103a410142665ac1ea3d83 columnFamilyName family 2024-12-05T14:13:31,934 DEBUG [StoreOpener-4edea3efe3103a410142665ac1ea3d83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,934 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b., pid=41, masterSystemTime=1733408011800 2024-12-05T14:13:31,934 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/59e732700952e1f67c04f79b391abf8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,935 INFO [StoreOpener-4edea3efe3103a410142665ac1ea3d83-1 {}] regionserver.HStore(327): Store=4edea3efe3103a410142665ac1ea3d83/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,935 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 59e732700952e1f67c04f79b391abf8c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60985920, jitterRate=-0.09123897552490234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,935 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:31,935 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,935 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 59e732700952e1f67c04f79b391abf8c: Running coprocessor pre-open hook at 1733408011909Writing region info on filesystem at 1733408011909Initializing all the Stores at 1733408011912 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011912Cleaning up temporary data from old regions at 1733408011924 (+12 ms)Running coprocessor post-open hooks at 1733408011935 (+11 ms)Region opened successfully at 1733408011935 2024-12-05T14:13:31,936 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,937 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c., pid=51, masterSystemTime=1733408011838 2024-12-05T14:13:31,937 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,938 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:31,938 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:31,938 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,938 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,938 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:31,939 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 79e2982faca0d2750624de91bb80c08e, NAME => 'test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-05T14:13:31,939 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,939 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN in 510 msec 2024-12-05T14:13:31,939 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,939 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,940 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:31,940 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:31,941 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:31,941 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => b645f799e60d0a296066576eac0b0285, NAME => 'test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-05T14:13:31,942 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,942 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=a24b719bda0e807d3ebf3858def7531b, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,942 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,942 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,942 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,942 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:31,946 INFO [StoreOpener-79e2982faca0d2750624de91bb80c08e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,946 INFO [StoreOpener-b645f799e60d0a296066576eac0b0285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:31,948 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/4edea3efe3103a410142665ac1ea3d83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,949 INFO [StoreOpener-b645f799e60d0a296066576eac0b0285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b645f799e60d0a296066576eac0b0285 columnFamilyName family 2024-12-05T14:13:31,949 DEBUG [StoreOpener-b645f799e60d0a296066576eac0b0285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,950 INFO [StoreOpener-79e2982faca0d2750624de91bb80c08e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79e2982faca0d2750624de91bb80c08e columnFamilyName family 2024-12-05T14:13:31,950 DEBUG [StoreOpener-79e2982faca0d2750624de91bb80c08e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,950 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 4edea3efe3103a410142665ac1ea3d83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64095130, jitterRate=-0.04490813612937927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,950 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:31,951 INFO [StoreOpener-b645f799e60d0a296066576eac0b0285-1 {}] regionserver.HStore(327): Store=b645f799e60d0a296066576eac0b0285/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,951 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 4edea3efe3103a410142665ac1ea3d83: Running coprocessor pre-open hook at 1733408011928Writing region info on filesystem at 1733408011928Initializing all the Stores at 1733408011930 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011930Cleaning up temporary data from old regions at 1733408011938 (+8 ms)Running coprocessor post-open hooks at 1733408011950 (+12 ms)Region opened successfully at 1733408011951 (+1 ms) 2024-12-05T14:13:31,951 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,952 INFO [StoreOpener-79e2982faca0d2750624de91bb80c08e-1 {}] regionserver.HStore(327): Store=79e2982faca0d2750624de91bb80c08e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,952 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59e732700952e1f67c04f79b391abf8c, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:31,953 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,953 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83., pid=34, masterSystemTime=1733408011750 2024-12-05T14:13:31,953 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,954 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,954 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,955 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,956 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,956 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:31,957 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:31,957 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,957 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:31,957 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,957 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 27fe33c5bc228b6793a2d277d2d7d9c2, NAME => 'test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-05T14:13:31,957 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,958 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,958 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,958 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=30 2024-12-05T14:13:31,960 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,36017,1733408008679, table=test, region=59e732700952e1f67c04f79b391abf8c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:31,960 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,36017,1733408008679, table=test, region=ffa1b4ffdfeddf082998369feea2e28b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:31,961 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=5c703cd272011369700660d18171711b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:31,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,40363,1733408008532 in 292 msec 2024-12-05T14:13:31,961 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=564487ebab3bd815a85a50262b275a91. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:31,962 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,963 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN in 538 msec 2024-12-05T14:13:31,965 INFO [StoreOpener-27fe33c5bc228b6793a2d277d2d7d9c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,967 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,969 INFO [StoreOpener-27fe33c5bc228b6793a2d277d2d7d9c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27fe33c5bc228b6793a2d277d2d7d9c2 columnFamilyName family 2024-12-05T14:13:31,969 DEBUG [StoreOpener-27fe33c5bc228b6793a2d277d2d7d9c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,970 INFO [StoreOpener-27fe33c5bc228b6793a2d277d2d7d9c2-1 {}] regionserver.HStore(327): Store=27fe33c5bc228b6793a2d277d2d7d9c2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,971 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4edea3efe3103a410142665ac1ea3d83, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:31,972 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,973 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,973 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,973 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,974 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/79e2982faca0d2750624de91bb80c08e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,974 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/b645f799e60d0a296066576eac0b0285/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,974 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened b645f799e60d0a296066576eac0b0285; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71321747, jitterRate=0.0627768486738205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,975 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:31,975 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for b645f799e60d0a296066576eac0b0285: Running coprocessor pre-open hook at 1733408011942Writing region info on filesystem at 1733408011942Initializing all the Stores at 1733408011944 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011944Cleaning up temporary data from old regions at 1733408011955 (+11 ms)Running coprocessor post-open hooks at 1733408011975 (+20 ms)Region opened successfully at 1733408011975 2024-12-05T14:13:31,975 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 79e2982faca0d2750624de91bb80c08e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61192165, jitterRate=-0.08816568553447723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,975 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:31,976 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285., pid=52, masterSystemTime=1733408011838 2024-12-05T14:13:31,976 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 79e2982faca0d2750624de91bb80c08e: Running coprocessor pre-open hook at 1733408011940Writing region info on filesystem at 1733408011940Initializing all the Stores at 1733408011942 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011942Cleaning up temporary data from old regions at 1733408011957 (+15 ms)Running coprocessor post-open hooks at 1733408011976 (+19 ms)Region opened successfully at 1733408011976 2024-12-05T14:13:31,976 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,978 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e., pid=38, masterSystemTime=1733408011800 2024-12-05T14:13:31,979 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:31,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=12 2024-12-05T14:13:31,979 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:31,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 59e732700952e1f67c04f79b391abf8c, server=218194e0d8ec,36017,1733408008679 in 255 msec 2024-12-05T14:13:31,980 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:31,980 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ffe5657e6841580f09ad342b0a1a3ad, NAME => 'test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-05T14:13:31,980 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,980 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,980 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,980 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,980 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/27fe33c5bc228b6793a2d277d2d7d9c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:31,981 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 27fe33c5bc228b6793a2d277d2d7d9c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59384359, jitterRate=-0.11510409414768219}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:31,982 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:31,982 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 27fe33c5bc228b6793a2d277d2d7d9c2: Running coprocessor pre-open hook at 1733408011958Writing region info on filesystem at 1733408011958Initializing all the Stores at 1733408011962 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011962Cleaning up temporary data from old regions at 1733408011973 (+11 ms)Running coprocessor post-open hooks at 1733408011982 (+9 ms)Region opened successfully at 1733408011982 2024-12-05T14:13:31,982 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:31,982 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:31,982 INFO [StoreOpener-0ffe5657e6841580f09ad342b0a1a3ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,983 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. 2024-12-05T14:13:31,983 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2., pid=42, masterSystemTime=1733408011750 2024-12-05T14:13:31,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=59e732700952e1f67c04f79b391abf8c, ASSIGN in 558 msec 2024-12-05T14:13:31,983 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 88e959e3bb59b49f279be7426309e052, NAME => 'test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-05T14:13:31,984 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:31,984 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:31,984 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:31,984 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:31,985 INFO [StoreOpener-0ffe5657e6841580f09ad342b0a1a3ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ffe5657e6841580f09ad342b0a1a3ad columnFamilyName family 2024-12-05T14:13:31,985 DEBUG [StoreOpener-0ffe5657e6841580f09ad342b0a1a3ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:31,986 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:31,986 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:31,986 INFO [StoreOpener-0ffe5657e6841580f09ad342b0a1a3ad-1 {}] regionserver.HStore(327): Store=0ffe5657e6841580f09ad342b0a1a3ad/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,987 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,989 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:31,989 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,990 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,990 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:31,993 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:31,993 INFO [StoreOpener-88e959e3bb59b49f279be7426309e052-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:31,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27fe33c5bc228b6793a2d277d2d7d9c2, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:31,996 INFO [StoreOpener-88e959e3bb59b49f279be7426309e052-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88e959e3bb59b49f279be7426309e052 columnFamilyName family 2024-12-05T14:13:31,997 DEBUG [StoreOpener-88e959e3bb59b49f279be7426309e052-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:31,997 INFO [StoreOpener-88e959e3bb59b49f279be7426309e052-1 {}] regionserver.HStore(327): Store=88e959e3bb59b49f279be7426309e052/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:31,998 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:31,999 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:31,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=28 2024-12-05T14:13:31,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,40363,1733408008532 in 350 msec 2024-12-05T14:13:31,999 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:32,000 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:32,000 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:32,001 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/0ffe5657e6841580f09ad342b0a1a3ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,002 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 0ffe5657e6841580f09ad342b0a1a3ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68101715, jitterRate=0.01479463279247284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,002 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:32,003 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 0ffe5657e6841580f09ad342b0a1a3ad: Running coprocessor pre-open hook at 1733408011981Writing region info on filesystem at 1733408011981Initializing all the Stores at 1733408011982 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011982Cleaning up temporary data from old regions at 1733408011990 (+8 ms)Running coprocessor post-open hooks at 1733408012002 (+12 ms)Region opened successfully at 1733408012002 2024-12-05T14:13:32,003 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:32,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, ASSIGN in 576 msec 2024-12-05T14:13:32,004 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad., pid=55, masterSystemTime=1733408011838 2024-12-05T14:13:32,006 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88e959e3bb59b49f279be7426309e052/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,007 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:32,007 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:32,008 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:32,008 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 88e959e3bb59b49f279be7426309e052; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62888153, jitterRate=-0.06289349496364594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,008 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:32,008 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 9dd338d2cdec8bcc235bdb212f336d28, NAME => 'test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-05T14:13:32,008 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 88e959e3bb59b49f279be7426309e052: Running coprocessor pre-open hook at 1733408011985Writing region info on filesystem at 1733408011985Initializing all the Stores at 1733408011986 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408011986Cleaning up temporary data from old regions at 1733408012000 (+14 ms)Running coprocessor post-open hooks at 1733408012008 (+8 ms)Region opened successfully at 1733408012008 2024-12-05T14:13:32,008 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,008 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:32,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=6 2024-12-05T14:13:32,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 27fe33c5bc228b6793a2d277d2d7d9c2, server=218194e0d8ec,45449,1733408008631 in 337 msec 2024-12-05T14:13:32,009 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,009 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,009 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052., pid=39, masterSystemTime=1733408011800 2024-12-05T14:13:32,012 INFO [StoreOpener-9dd338d2cdec8bcc235bdb212f336d28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,012 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. 2024-12-05T14:13:32,012 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. 2024-12-05T14:13:32,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=27fe33c5bc228b6793a2d277d2d7d9c2, ASSIGN in 587 msec 2024-12-05T14:13:32,013 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:32,013 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => aba2432d09baa11b4c1ea7026329d127, NAME => 'test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-05T14:13:32,013 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,013 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:32,014 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,014 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,015 INFO [StoreOpener-9dd338d2cdec8bcc235bdb212f336d28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9dd338d2cdec8bcc235bdb212f336d28 columnFamilyName family 2024-12-05T14:13:32,016 DEBUG [StoreOpener-9dd338d2cdec8bcc235bdb212f336d28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:32,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=88e959e3bb59b49f279be7426309e052, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:32,016 INFO [StoreOpener-aba2432d09baa11b4c1ea7026329d127-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,016 INFO [StoreOpener-9dd338d2cdec8bcc235bdb212f336d28-1 {}] regionserver.HStore(327): Store=9dd338d2cdec8bcc235bdb212f336d28/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:32,016 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,018 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,019 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,019 INFO [StoreOpener-aba2432d09baa11b4c1ea7026329d127-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aba2432d09baa11b4c1ea7026329d127 columnFamilyName family 2024-12-05T14:13:32,019 DEBUG [StoreOpener-aba2432d09baa11b4c1ea7026329d127-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:32,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:32,020 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,020 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,020 INFO [StoreOpener-aba2432d09baa11b4c1ea7026329d127-1 {}] regionserver.HStore(327): Store=aba2432d09baa11b4c1ea7026329d127/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:32,021 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,021 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:32,022 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,022 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,022 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,023 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,023 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ffe5657e6841580f09ad342b0a1a3ad, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:32,026 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,027 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/9dd338d2cdec8bcc235bdb212f336d28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,029 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 9dd338d2cdec8bcc235bdb212f336d28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65887901, jitterRate=-0.018193766474723816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,029 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:32,029 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 9dd338d2cdec8bcc235bdb212f336d28: Running coprocessor pre-open hook at 1733408012009Writing region info on filesystem at 1733408012009Initializing all the Stores at 1733408012011 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408012011Cleaning up temporary data from old regions at 1733408012020 (+9 ms)Running coprocessor post-open hooks at 1733408012029 (+9 ms)Region opened successfully at 1733408012029 2024-12-05T14:13:32,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=21 2024-12-05T14:13:32,030 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28., pid=53, masterSystemTime=1733408011838 2024-12-05T14:13:32,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,40363,1733408008532 in 373 msec 2024-12-05T14:13:32,030 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/aba2432d09baa11b4c1ea7026329d127/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,031 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened aba2432d09baa11b4c1ea7026329d127; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64603757, jitterRate=-0.03732900321483612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,031 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:32,031 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for aba2432d09baa11b4c1ea7026329d127: Running coprocessor pre-open hook at 1733408012014Writing region info on filesystem at 1733408012014Initializing all the Stores at 1733408012015 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408012015Cleaning up temporary data from old regions at 1733408012023 (+8 ms)Running coprocessor post-open hooks at 1733408012031 (+8 ms)Region opened successfully at 1733408012031 2024-12-05T14:13:32,032 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127., pid=47, masterSystemTime=1733408011800 2024-12-05T14:13:32,033 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:32,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN in 608 msec 2024-12-05T14:13:32,034 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:32,034 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:32,034 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => c14601cce05aed77304a5058eff93ba1, NAME => 'test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-05T14:13:32,035 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,035 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:32,035 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,035 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,037 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:32,037 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:32,037 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:32,037 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:32,037 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 811fb81e434a8467b09b0f283ae0207b, NAME => 'test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-05T14:13:32,038 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,038 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:32,038 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,038 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dd338d2cdec8bcc235bdb212f336d28, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:32,042 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=aba2432d09baa11b4c1ea7026329d127, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:32,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba2432d09baa11b4c1ea7026329d127, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:32,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=19 2024-12-05T14:13:32,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 9dd338d2cdec8bcc235bdb212f336d28, server=218194e0d8ec,36017,1733408008679 in 322 msec 2024-12-05T14:13:32,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9dd338d2cdec8bcc235bdb212f336d28, ASSIGN in 627 msec 2024-12-05T14:13:32,052 INFO [StoreOpener-c14601cce05aed77304a5058eff93ba1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,052 INFO [StoreOpener-811fb81e434a8467b09b0f283ae0207b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,054 INFO [StoreOpener-811fb81e434a8467b09b0f283ae0207b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 811fb81e434a8467b09b0f283ae0207b columnFamilyName family 2024-12-05T14:13:32,055 DEBUG [StoreOpener-811fb81e434a8467b09b0f283ae0207b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:32,055 INFO [StoreOpener-811fb81e434a8467b09b0f283ae0207b-1 {}] regionserver.HStore(327): Store=811fb81e434a8467b09b0f283ae0207b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:32,056 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,057 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=11 2024-12-05T14:13:32,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure aba2432d09baa11b4c1ea7026329d127, server=218194e0d8ec,40363,1733408008532 in 357 msec 2024-12-05T14:13:32,058 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,059 INFO [StoreOpener-c14601cce05aed77304a5058eff93ba1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c14601cce05aed77304a5058eff93ba1 columnFamilyName family 2024-12-05T14:13:32,059 DEBUG [StoreOpener-c14601cce05aed77304a5058eff93ba1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:32,059 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,059 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN in 636 msec 2024-12-05T14:13:32,061 INFO [StoreOpener-c14601cce05aed77304a5058eff93ba1-1 {}] regionserver.HStore(327): Store=c14601cce05aed77304a5058eff93ba1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:32,061 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,063 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,064 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,36017,1733408008679, table=test, region=ffa1b4ffdfeddf082998369feea2e28b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:32,064 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,065 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,065 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,065 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=8 2024-12-05T14:13:32,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ffe5657e6841580f09ad342b0a1a3ad, server=218194e0d8ec,36017,1733408008679 in 330 msec 2024-12-05T14:13:32,069 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,072 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/811fb81e434a8467b09b0f283ae0207b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0ffe5657e6841580f09ad342b0a1a3ad, ASSIGN in 646 msec 2024-12-05T14:13:32,073 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 811fb81e434a8467b09b0f283ae0207b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61646748, jitterRate=-0.08139187097549438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,073 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:32,073 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 811fb81e434a8467b09b0f283ae0207b: Running coprocessor pre-open hook at 1733408012038Writing region info on filesystem at 1733408012038Initializing all the Stores at 1733408012040 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408012040Cleaning up temporary data from old regions at 1733408012059 (+19 ms)Running coprocessor post-open hooks at 1733408012073 (+14 ms)Region opened successfully at 1733408012073 2024-12-05T14:13:32,074 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b., pid=48, masterSystemTime=1733408011800 2024-12-05T14:13:32,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=b645f799e60d0a296066576eac0b0285, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:32,080 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=5c703cd272011369700660d18171711b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:32,080 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=4edea3efe3103a410142665ac1ea3d83. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:32,080 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=564487ebab3bd815a85a50262b275a91. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:32,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure b645f799e60d0a296066576eac0b0285, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:32,081 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:32,082 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:32,082 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:32,082 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=811fb81e434a8467b09b0f283ae0207b, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:32,082 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => c129ce5c0755b4946bdefc2d36f50149, NAME => 'test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-05T14:13:32,083 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c14601cce05aed77304a5058eff93ba1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,083 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,083 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:32,084 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,084 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,084 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened c14601cce05aed77304a5058eff93ba1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60867789, jitterRate=-0.09299926459789276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,084 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:32,085 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for c14601cce05aed77304a5058eff93ba1: Running coprocessor pre-open hook at 1733408012035Writing region info on filesystem at 1733408012035Initializing all the Stores at 1733408012037 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408012037Cleaning up temporary data from old regions at 1733408012065 (+28 ms)Running coprocessor post-open hooks at 1733408012084 (+19 ms)Region opened successfully at 1733408012084 2024-12-05T14:13:32,086 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1., pid=49, masterSystemTime=1733408011838 2024-12-05T14:13:32,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:32,088 INFO [StoreOpener-c129ce5c0755b4946bdefc2d36f50149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,090 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:32,090 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:32,090 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:32,090 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 70c6030455907dcfb74337bc850642ff, NAME => 'test,sss,1733408011017.70c6030455907dcfb74337bc850642ff.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-05T14:13:32,091 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,091 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,sss,1733408011017.70c6030455907dcfb74337bc850642ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:32,091 INFO [StoreOpener-c129ce5c0755b4946bdefc2d36f50149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c129ce5c0755b4946bdefc2d36f50149 columnFamilyName family 2024-12-05T14:13:32,091 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,091 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,091 DEBUG [StoreOpener-c129ce5c0755b4946bdefc2d36f50149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:32,092 INFO [StoreOpener-c129ce5c0755b4946bdefc2d36f50149-1 {}] regionserver.HStore(327): Store=c129ce5c0755b4946bdefc2d36f50149/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:32,092 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,093 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,093 INFO [StoreOpener-70c6030455907dcfb74337bc850642ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,094 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,095 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,095 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=14 2024-12-05T14:13:32,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure b645f799e60d0a296066576eac0b0285, server=218194e0d8ec,36017,1733408008679 in 369 msec 2024-12-05T14:13:32,098 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,100 INFO [StoreOpener-70c6030455907dcfb74337bc850642ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70c6030455907dcfb74337bc850642ff columnFamilyName family 2024-12-05T14:13:32,100 DEBUG [StoreOpener-70c6030455907dcfb74337bc850642ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:32,101 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c129ce5c0755b4946bdefc2d36f50149/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b645f799e60d0a296066576eac0b0285, ASSIGN in 673 msec 2024-12-05T14:13:32,101 INFO [StoreOpener-70c6030455907dcfb74337bc850642ff-1 {}] regionserver.HStore(327): Store=70c6030455907dcfb74337bc850642ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:32,101 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,101 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened c129ce5c0755b4946bdefc2d36f50149; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58939049, jitterRate=-0.12173973023891449}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,102 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:32,102 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for c129ce5c0755b4946bdefc2d36f50149: Running coprocessor pre-open hook at 1733408012084Writing region info on filesystem at 1733408012084Initializing all the Stores at 1733408012087 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408012087Cleaning up temporary data from old regions at 1733408012095 (+8 ms)Running coprocessor post-open hooks at 1733408012102 (+7 ms)Region opened successfully at 1733408012102 2024-12-05T14:13:32,103 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:32,103 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149., pid=43, masterSystemTime=1733408011800 2024-12-05T14:13:32,103 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,104 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,105 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c14601cce05aed77304a5058eff93ba1, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:32,108 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,109 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:32,109 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:32,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=15 2024-12-05T14:13:32,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,40363,1733408008532 in 409 msec 2024-12-05T14:13:32,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN in 691 msec 2024-12-05T14:13:32,116 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c129ce5c0755b4946bdefc2d36f50149, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:32,117 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/70c6030455907dcfb74337bc850642ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,119 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 70c6030455907dcfb74337bc850642ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62403105, jitterRate=-0.07012127339839935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,119 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:32,119 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 70c6030455907dcfb74337bc850642ff: Running coprocessor pre-open hook at 1733408012091Writing region info on filesystem at 1733408012091Initializing all the Stores at 1733408012092 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408012092Cleaning up temporary data from old regions at 1733408012105 (+13 ms)Running coprocessor post-open hooks at 1733408012119 (+14 ms)Region opened successfully at 1733408012119 2024-12-05T14:13:32,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:32,121 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733408011017.70c6030455907dcfb74337bc850642ff., pid=54, masterSystemTime=1733408011838 2024-12-05T14:13:32,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=26 2024-12-05T14:13:32,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure c14601cce05aed77304a5058eff93ba1, server=218194e0d8ec,36017,1733408008679 in 412 msec 2024-12-05T14:13:32,124 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:32,124 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:32,124 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:32,124 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 88bd874c8e3a659802f7bbb662036a12, NAME => 'test,,1733408011017.88bd874c8e3a659802f7bbb662036a12.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-05T14:13:32,125 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,125 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,,1733408011017.88bd874c8e3a659802f7bbb662036a12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:32,125 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,125 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c14601cce05aed77304a5058eff93ba1, ASSIGN in 701 msec 2024-12-05T14:13:32,127 INFO [StoreOpener-88bd874c8e3a659802f7bbb662036a12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=70c6030455907dcfb74337bc850642ff, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:32,130 INFO [StoreOpener-88bd874c8e3a659802f7bbb662036a12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bd874c8e3a659802f7bbb662036a12 columnFamilyName family 2024-12-05T14:13:32,130 DEBUG [StoreOpener-88bd874c8e3a659802f7bbb662036a12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:32,131 INFO [StoreOpener-88bd874c8e3a659802f7bbb662036a12-1 {}] regionserver.HStore(327): Store=88bd874c8e3a659802f7bbb662036a12/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:32,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 70c6030455907dcfb74337bc850642ff, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:32,132 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,133 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,133 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,134 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,134 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=18 2024-12-05T14:13:32,137 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,40363,1733408008532 in 457 msec 2024-12-05T14:13:32,140 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88bd874c8e3a659802f7bbb662036a12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN in 715 msec 2024-12-05T14:13:32,142 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 88bd874c8e3a659802f7bbb662036a12; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74099391, jitterRate=0.10416696965694427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,142 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:32,142 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 88bd874c8e3a659802f7bbb662036a12: Running coprocessor pre-open hook at 1733408012125Writing region info on filesystem at 1733408012125Initializing all the Stores at 1733408012126 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408012126Cleaning up temporary data from old regions at 1733408012134 (+8 ms)Running coprocessor post-open hooks at 1733408012142 (+8 ms)Region opened successfully at 1733408012142 2024-12-05T14:13:32,144 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733408011017.88bd874c8e3a659802f7bbb662036a12., pid=56, masterSystemTime=1733408011838 2024-12-05T14:13:32,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=23 2024-12-05T14:13:32,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 70c6030455907dcfb74337bc850642ff, server=218194e0d8ec,36017,1733408008679 in 415 msec 2024-12-05T14:13:32,146 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:32,146 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:32,147 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:32,147 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 710e86e039ffc8d96f214260888d4849, NAME => 'test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-05T14:13:32,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:32,147 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,147 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:32,147 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,148 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,150 INFO [StoreOpener-710e86e039ffc8d96f214260888d4849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bd874c8e3a659802f7bbb662036a12, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:32,154 INFO [StoreOpener-710e86e039ffc8d96f214260888d4849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 710e86e039ffc8d96f214260888d4849 columnFamilyName family 2024-12-05T14:13:32,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=70c6030455907dcfb74337bc850642ff, ASSIGN in 727 msec 2024-12-05T14:13:32,154 DEBUG [StoreOpener-710e86e039ffc8d96f214260888d4849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:32,155 INFO [StoreOpener-710e86e039ffc8d96f214260888d4849-1 {}] regionserver.HStore(327): Store=710e86e039ffc8d96f214260888d4849/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:32,155 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,156 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,157 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,158 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,158 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=5 2024-12-05T14:13:32,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 88bd874c8e3a659802f7bbb662036a12, server=218194e0d8ec,36017,1733408008679 in 419 msec 2024-12-05T14:13:32,162 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=88bd874c8e3a659802f7bbb662036a12, ASSIGN in 738 msec 2024-12-05T14:13:32,169 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/710e86e039ffc8d96f214260888d4849/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T14:13:32,169 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,36017,1733408008679, table=test, region=ffa1b4ffdfeddf082998369feea2e28b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:32,170 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 710e86e039ffc8d96f214260888d4849; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59319936, jitterRate=-0.11606407165527344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:32,170 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:32,170 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 710e86e039ffc8d96f214260888d4849: Running coprocessor pre-open hook at 1733408012148Writing region info on filesystem at 1733408012148Initializing all the Stores at 1733408012149 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408012150 (+1 ms)Cleaning up temporary data from old regions at 1733408012158 (+8 ms)Running coprocessor post-open hooks at 1733408012170 (+12 ms)Region opened successfully at 1733408012170 2024-12-05T14:13:32,171 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849., pid=45, masterSystemTime=1733408011838 2024-12-05T14:13:32,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=16 2024-12-05T14:13:32,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 4edea3efe3103a410142665ac1ea3d83, server=218194e0d8ec,45449,1733408008631 in 549 msec 2024-12-05T14:13:32,175 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:32,175 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:32,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4edea3efe3103a410142665ac1ea3d83, ASSIGN in 751 msec 2024-12-05T14:13:32,177 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=710e86e039ffc8d96f214260888d4849, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:32,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 710e86e039ffc8d96f214260888d4849, server=218194e0d8ec,36017,1733408008679 because future has completed 2024-12-05T14:13:32,181 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, openSeqNum=2, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:32,184 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=5c703cd272011369700660d18171711b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:32,184 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=ce07d807a24e01c939692ade9b01d97e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:32,184 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,45449,1733408008631, table=test, region=564487ebab3bd815a85a50262b275a91. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:32,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce07d807a24e01c939692ade9b01d97e, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:13:32,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=29 2024-12-05T14:13:32,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 710e86e039ffc8d96f214260888d4849, server=218194e0d8ec,36017,1733408008679 in 500 msec 2024-12-05T14:13:32,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=710e86e039ffc8d96f214260888d4849, ASSIGN in 769 msec 2024-12-05T14:13:32,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-12-05T14:13:32,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure ce07d807a24e01c939692ade9b01d97e, server=218194e0d8ec,45449,1733408008631 in 593 msec 2024-12-05T14:13:32,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ce07d807a24e01c939692ade9b01d97e, ASSIGN in 777 msec 2024-12-05T14:13:32,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=17 2024-12-05T14:13:32,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure ffa1b4ffdfeddf082998369feea2e28b, server=218194e0d8ec,36017,1733408008679 in 492 msec 2024-12-05T14:13:32,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ffa1b4ffdfeddf082998369feea2e28b, ASSIGN in 784 msec 2024-12-05T14:13:32,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=9 2024-12-05T14:13:32,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 5c703cd272011369700660d18171711b, server=218194e0d8ec,45449,1733408008631 in 572 msec 2024-12-05T14:13:32,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5c703cd272011369700660d18171711b, ASSIGN in 790 msec 2024-12-05T14:13:32,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=25 2024-12-05T14:13:32,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 564487ebab3bd815a85a50262b275a91, server=218194e0d8ec,45449,1733408008631 in 589 msec 2024-12-05T14:13:32,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=4 2024-12-05T14:13:32,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=564487ebab3bd815a85a50262b275a91, ASSIGN in 797 msec 2024-12-05T14:13:32,226 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T14:13:32,227 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733408012227"}]},"ts":"1733408012227"} 2024-12-05T14:13:32,230 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-05T14:13:32,231 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T14:13:32,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.2070 sec 2024-12-05T14:13:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T14:13:33,202 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-05T14:13:33,202 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-05T14:13:33,205 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T14:13:33,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-05T14:13:33,226 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T14:13:33,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-05T14:13:33,228 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T14:13:33,234 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-05T14:13:33,257 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-05T14:13:33,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T14:13:33,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T14:13:33,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T14:13:33,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T14:13:33,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 88bd874c8e3a659802f7bbb662036a12 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0ffe5657e6841580f09ad342b0a1a3ad changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 59e732700952e1f67c04f79b391abf8c changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b645f799e60d0a296066576eac0b0285 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ffa1b4ffdfeddf082998369feea2e28b changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9dd338d2cdec8bcc235bdb212f336d28 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 70c6030455907dcfb74337bc850642ff changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c14601cce05aed77304a5058eff93ba1 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 710e86e039ffc8d96f214260888d4849 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 894d0c86da04798f8fea0640c4aa6d05 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region abd1856616590a0391ff54d53cfbbb53 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region aba2432d09baa11b4c1ea7026329d127 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 811fb81e434a8467b09b0f283ae0207b changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c129ce5c0755b4946bdefc2d36f50149 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 88e959e3bb59b49f279be7426309e052 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cd727192dbe84371a93aad39b906dd3b changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 79e2982faca0d2750624de91bb80c08e changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a24b719bda0e807d3ebf3858def7531b changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 27fe33c5bc228b6793a2d277d2d7d9c2 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5c703cd272011369700660d18171711b changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3fe46a05dda7cc73743c5e741892f632 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4edea3efe3103a410142665ac1ea3d83 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ce07d807a24e01c939692ade9b01d97e changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 44e3fd15a9de4a8ceefe8e8505a163f2 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 564487ebab3bd815a85a50262b275a91 changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b620ca010e58c2eb2f2309f4a11c2dba changed from -1.0 to 0.0, refreshing cache 2024-12-05T14:13:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-05T14:13:33,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-05T14:13:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-05T14:13:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(204): Hosts are {218194e0d8ec=0} racks are {/default-rack=0} 2024-12-05T14:13:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-12-05T14:13:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-12-05T14:13:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-12-05T14:13:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T14:13:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T14:13:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-05T14:13:33,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T14:13:33,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T14:13:33,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-05T14:13:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-05T14:13:33,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-12-05T14:13:33,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-12-05T14:13:33,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 444 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-05T14:13:33,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-12-05T14:13:33,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.HMaster(2172): balance hri=79e2982faca0d2750624de91bb80c08e, source=218194e0d8ec,40363,1733408008532, destination=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:33,798 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:33,799 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=79e2982faca0d2750624de91bb80c08e, regionState=CLOSING, regionLocation=218194e0d8ec,40363,1733408008532 2024-12-05T14:13:33,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE because future has completed 2024-12-05T14:13:33,802 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T14:13:33,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,40363,1733408008532}] 2024-12-05T14:13:33,960 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:33,960 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T14:13:33,961 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 79e2982faca0d2750624de91bb80c08e, disabling compactions & flushes 2024-12-05T14:13:33,961 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:33,962 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:33,962 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. after waiting 0 ms 2024-12-05T14:13:33,962 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:33,969 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/79e2982faca0d2750624de91bb80c08e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:33,973 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:33,973 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 79e2982faca0d2750624de91bb80c08e: Waiting for close lock at 1733408013961Running coprocessor pre-close hooks at 1733408013961Disabling compacts and flushes for region at 1733408013961Disabling writes for close at 1733408013962 (+1 ms)Writing region close event to WAL at 1733408013962Running coprocessor post-close hooks at 1733408013971 (+9 ms)Closed at 1733408013973 (+2 ms) 2024-12-05T14:13:33,974 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 79e2982faca0d2750624de91bb80c08e move to 218194e0d8ec,45449,1733408008631 record at close sequenceid=2 2024-12-05T14:13:33,978 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:33,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=79e2982faca0d2750624de91bb80c08e, regionState=CLOSED 2024-12-05T14:13:33,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,40363,1733408008532 because future has completed 2024-12-05T14:13:33,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-05T14:13:33,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,40363,1733408008532 in 182 msec 2024-12-05T14:13:33,988 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE; state=CLOSED, location=218194e0d8ec,45449,1733408008631; forceNewPlan=false, retain=false 2024-12-05T14:13:34,139 INFO [218194e0d8ec:38561 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T14:13:34,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=79e2982faca0d2750624de91bb80c08e, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:34,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE because future has completed 2024-12-05T14:13:34,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:34,307 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:34,307 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 79e2982faca0d2750624de91bb80c08e, NAME => 'test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-05T14:13:34,308 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,308 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:34,308 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,308 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,310 INFO [StoreOpener-79e2982faca0d2750624de91bb80c08e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,311 INFO [StoreOpener-79e2982faca0d2750624de91bb80c08e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79e2982faca0d2750624de91bb80c08e columnFamilyName family 2024-12-05T14:13:34,312 DEBUG [StoreOpener-79e2982faca0d2750624de91bb80c08e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:34,312 INFO [StoreOpener-79e2982faca0d2750624de91bb80c08e-1 {}] regionserver.HStore(327): Store=79e2982faca0d2750624de91bb80c08e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:34,312 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,314 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,315 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,316 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,316 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,318 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,319 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 79e2982faca0d2750624de91bb80c08e; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63836205, jitterRate=-0.04876641929149628}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:34,319 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:34,319 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 79e2982faca0d2750624de91bb80c08e: Running coprocessor pre-open hook at 1733408014309Writing region info on filesystem at 1733408014309Initializing all the Stores at 1733408014310 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408014310Cleaning up temporary data from old regions at 1733408014316 (+6 ms)Running coprocessor post-open hooks at 1733408014319 (+3 ms)Region opened successfully at 1733408014319 2024-12-05T14:13:34,320 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e., pid=59, masterSystemTime=1733408014295 2024-12-05T14:13:34,323 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:34,323 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:34,325 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, openSeqNum=5, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:34,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,45449,1733408008631 because future has completed 2024-12-05T14:13:34,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-12-05T14:13:34,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 79e2982faca0d2750624de91bb80c08e, server=218194e0d8ec,45449,1733408008631 in 187 msec 2024-12-05T14:13:34,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE in 537 msec 2024-12-05T14:13:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-05T14:13:34,400 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T14:13:34,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-12-05T14:13:34,421 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=382, OpenFileDescriptor=645, MaxFileDescriptor=1048576, SystemLoadAverage=392, ProcessCount=11, AvailableMemoryMB=6289 2024-12-05T14:13:34,424 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-05T14:13:34,425 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-05T14:13:34,425 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 218194e0d8ec,40363,1733408008532 2024-12-05T14:13:34,426 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 218194e0d8ec,40363,1733408008532: Simulated kill ***** 2024-12-05T14:13:34,426 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-05T14:13:34,428 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-05T14:13:34,431 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-05T14:13:34,431 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-05T14:13:34,431 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-05T14:13:34,431 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 341585280 }, "NonHeapMemoryUsage": { "committed": 154468352, "init": 7667712, "max": -1, "used": 151923920 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "218194e0d8ec" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "218194e0d8ec" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "218194e0d8ec" } ] } 2024-12-05T14:13:34,434 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38561 {}] master.MasterRpcServices(700): 218194e0d8ec,40363,1733408008532 reported a fatal error: ***** ABORTING region server 218194e0d8ec,40363,1733408008532: Simulated kill ***** 2024-12-05T14:13:34,436 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '218194e0d8ec,40363,1733408008532' ***** 2024-12-05T14:13:34,436 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-05T14:13:34,437 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T14:13:34,437 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T14:13:34,437 INFO [RS:0;218194e0d8ec:40363 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-05T14:13:34,437 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T14:13:34,437 INFO [RS:0;218194e0d8ec:40363 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-05T14:13:34,437 INFO [RS:0;218194e0d8ec:40363 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T14:13:34,437 INFO [RS:0;218194e0d8ec:40363 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;218194e0d8ec:40363. 2024-12-05T14:13:34,437 DEBUG [RS:0;218194e0d8ec:40363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T14:13:34,441 DEBUG [RS:0;218194e0d8ec:40363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:34,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741836_1012 (size=33749) 2024-12-05T14:13:34,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741836_1012 (size=33749) 2024-12-05T14:13:34,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741836_1012 (size=33749) 2024-12-05T14:13:34,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741835_1011 (size=2774) 2024-12-05T14:13:34,455 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532/218194e0d8ec%2C40363%2C1733408008532.1733408010132 not finished, retry = 0 2024-12-05T14:13:34,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741835_1011 (size=2774) 2024-12-05T14:13:34,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741835_1011 (size=2774) 2024-12-05T14:13:34,466 INFO [regionserver/218194e0d8ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T14:13:34,557 DEBUG [RS:0;218194e0d8ec:40363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:34,557 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T14:13:34,557 INFO [RS:0;218194e0d8ec:40363 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40363 2024-12-05T14:13:34,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/218194e0d8ec,40363,1733408008532 2024-12-05T14:13:34,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T14:13:34,625 INFO [RS:0;218194e0d8ec:40363 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T14:13:34,636 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [218194e0d8ec,40363,1733408008532] 2024-12-05T14:13:34,646 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/218194e0d8ec,40363,1733408008532 already deleted, retry=false 2024-12-05T14:13:34,646 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 218194e0d8ec,40363,1733408008532 on 218194e0d8ec,38561,1733408007726 2024-12-05T14:13:34,654 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:34,658 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 218194e0d8ec,40363,1733408008532 (carryingMeta=true) 218194e0d8ec,40363,1733408008532/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@76c9bec4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-05T14:13:34,660 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:34,661 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true, isMeta: true 2024-12-05T14:13:34,664 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting 2024-12-05T14:13:34,665 INFO [PEWorker-1 {}] master.SplitWALManager(105): 218194e0d8ec,40363,1733408008532 WAL count=1, meta=true 2024-12-05T14:13:34,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta}] 2024-12-05T14:13:34,673 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:34,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:34,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T14:13:34,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40363-0x101a5d71eb70001, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T14:13:34,736 INFO [RS:0;218194e0d8ec:40363 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T14:13:34,737 INFO [RS:0;218194e0d8ec:40363 {}] regionserver.HRegionServer(1031): Exiting; stopping=218194e0d8ec,40363,1733408008532; zookeeper connection closed. 2024-12-05T14:13:34,737 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54fba5f9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54fba5f9 2024-12-05T14:13:34,738 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T14:13:34,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36017 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-12-05T14:13:34,864 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, size=33.0 K (33749bytes) 2024-12-05T14:13:34,864 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-05T14:13:34,864 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-05T14:13:34,865 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta 2024-12-05T14:13:34,870 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta after 3ms 2024-12-05T14:13:34,878 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta took 16ms 2024-12-05T14:13:34,886 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-05T14:13:34,892 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta.temp 2024-12-05T14:13:34,893 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta so closing down 2024-12-05T14:13:34,893 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T14:13:34,895 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta.temp 2024-12-05T14:13:34,899 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T14:13:34,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741864_1040 (size=33508) 2024-12-05T14:13:34,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741864_1040 (size=33508) 2024-12-05T14:13:34,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741864_1040 (size=33508) 2024-12-05T14:13:34,909 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) 2024-12-05T14:13:34,911 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta.temp to hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-05T14:13:34,915 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 32 ms; skipped=1; WAL=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-12-05T14:13:34,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, journal: Splitting hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, size=33.0 K (33749bytes) at 1733408014864Finishing writing output for hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta so closing down at 1733408014893 (+29 ms)Creating recovered edits writer path=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta.temp at 1733408014895 (+2 ms)3 split writer threads finished at 1733408014899 (+4 ms)Closed recovered edits writer path=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) at 1733408014909 (+10 ms)Rename recovered edits hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta.temp to hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1733408014911 (+2 ms)Processed 62 edits across 1 Regions in 32 ms; skipped=1; WAL=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1733408014915 (+4 ms) 2024-12-05T14:13:34,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta 2024-12-05T14:13:34,917 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-05T14:13:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-05T14:13:34,926 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta to hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/oldWALs 2024-12-05T14:13:34,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-05T14:13:34,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 in 251 msec 2024-12-05T14:13:34,930 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:34,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T14:13:34,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 in 263 msec 2024-12-05T14:13:34,936 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting dir is empty, no logs to split. 2024-12-05T14:13:34,936 INFO [PEWorker-1 {}] master.SplitWALManager(105): 218194e0d8ec,40363,1733408008532 WAL count=0, meta=true 2024-12-05T14:13:34,936 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 218194e0d8ec,40363,1733408008532 WAL splitting is done? wals=0, meta=true 2024-12-05T14:13:34,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T14:13:34,940 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:34,941 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:35,035 INFO [regionserver/218194e0d8ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T14:13:35,035 INFO [regionserver/218194e0d8ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T14:13:35,092 DEBUG [218194e0d8ec:38561 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T14:13:35,093 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(204): Hosts are {218194e0d8ec=0} racks are {/default-rack=0} 2024-12-05T14:13:35,093 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T14:13:35,093 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T14:13:35,094 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T14:13:35,094 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T14:13:35,094 INFO [218194e0d8ec:38561 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T14:13:35,094 INFO [218194e0d8ec:38561 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T14:13:35,094 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T14:13:35,095 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:35,100 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPENING 2024-12-05T14:13:35,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:35,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:35,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:35,120 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:35,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:35,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:35,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:35,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:35,280 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T14:13:35,280 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T14:13:35,280 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T14:13:35,283 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=218194e0d8ec%2C36017%2C1733408008679.meta, suffix=.meta, logDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,36017,1733408008679, archiveDir=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/oldWALs, maxLogs=32 2024-12-05T14:13:35,301 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,36017,1733408008679/218194e0d8ec%2C36017%2C1733408008679.meta.1733408015286.meta, exclude list is [], retry=0 2024-12-05T14:13:35,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44997,DS-0e75615e-d195-4475-b542-d756761e1516,DISK] 2024-12-05T14:13:35,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32959,DS-7c4ddf2e-589d-4886-a4d7-a998c1b0e99b,DISK] 2024-12-05T14:13:35,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38807,DS-a9dc01fb-9497-4dc5-b5a2-ed0f56e6f176,DISK] 2024-12-05T14:13:35,306 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,36017,1733408008679/218194e0d8ec%2C36017%2C1733408008679.meta.1733408015286.meta 2024-12-05T14:13:35,306 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33877:33877),(127.0.0.1/127.0.0.1:35745:35745),(127.0.0.1/127.0.0.1:35641:35641)] 2024-12-05T14:13:35,306 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T14:13:35,307 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T14:13:35,307 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T14:13:35,307 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T14:13:35,307 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T14:13:35,307 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:35,307 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T14:13:35,307 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T14:13:35,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T14:13:35,310 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T14:13:35,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:35,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:35,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T14:13:35,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T14:13:35,312 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:35,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:35,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T14:13:35,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T14:13:35,313 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:35,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:35,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T14:13:35,315 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T14:13:35,315 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:35,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T14:13:35,316 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T14:13:35,317 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740 2024-12-05T14:13:35,318 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740 2024-12-05T14:13:35,320 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-05T14:13:35,332 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-05T14:13:35,335 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-12-05T14:13:35,399 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/info/53fc85f5558f4ec3a6946fda3e586ee8 is 138, key is test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2./info:regioninfo/1733408011990/Put/seqid=0 2024-12-05T14:13:35,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741866_1042 (size=36376) 2024-12-05T14:13:35,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741866_1042 (size=36376) 2024-12-05T14:13:35,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741866_1042 (size=36376) 2024-12-05T14:13:35,416 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/info/53fc85f5558f4ec3a6946fda3e586ee8 2024-12-05T14:13:35,477 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/ns/48bbe0b9d9494dfc816eac43b46360ce is 43, key is default/ns:d/1733408010810/Put/seqid=0 2024-12-05T14:13:35,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741867_1043 (size=5153) 2024-12-05T14:13:35,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741867_1043 (size=5153) 2024-12-05T14:13:35,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741867_1043 (size=5153) 2024-12-05T14:13:35,487 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/ns/48bbe0b9d9494dfc816eac43b46360ce 2024-12-05T14:13:35,509 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/table/b18a40cf7767401ea6fb06d9f39a73bb is 40, key is test/table:state/1733408012227/Put/seqid=0 2024-12-05T14:13:35,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741868_1044 (size=5165) 2024-12-05T14:13:35,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741868_1044 (size=5165) 2024-12-05T14:13:35,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741868_1044 (size=5165) 2024-12-05T14:13:35,522 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/table/b18a40cf7767401ea6fb06d9f39a73bb 2024-12-05T14:13:35,531 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/info/53fc85f5558f4ec3a6946fda3e586ee8 as hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/info/53fc85f5558f4ec3a6946fda3e586ee8 2024-12-05T14:13:35,540 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/info/53fc85f5558f4ec3a6946fda3e586ee8, entries=273, sequenceid=64, filesize=35.5 K 2024-12-05T14:13:35,542 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/ns/48bbe0b9d9494dfc816eac43b46360ce as hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/ns/48bbe0b9d9494dfc816eac43b46360ce 2024-12-05T14:13:35,553 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/ns/48bbe0b9d9494dfc816eac43b46360ce, entries=2, sequenceid=64, filesize=5.0 K 2024-12-05T14:13:35,554 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/table/b18a40cf7767401ea6fb06d9f39a73bb as hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/table/b18a40cf7767401ea6fb06d9f39a73bb 2024-12-05T14:13:35,563 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/table/b18a40cf7767401ea6fb06d9f39a73bb, entries=2, sequenceid=64, filesize=5.0 K 2024-12-05T14:13:35,567 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 232ms, sequenceid=64, compaction requested=false; wal=null 2024-12-05T14:13:35,568 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T14:13:35,571 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-05T14:13:35,572 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T14:13:35,572 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T14:13:35,573 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T14:13:35,576 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T14:13:35,579 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-12-05T14:13:35,580 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73107079, jitterRate=0.08938036859035492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T14:13:35,580 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T14:13:35,580 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733408015307Writing region info on filesystem at 1733408015307Initializing all the Stores at 1733408015309 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408015309Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408015309Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408015309Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733408015309Obtaining lock to block concurrent updates at 1733408015335 (+26 ms)Preparing flush snapshotting stores in 1588230740 at 1733408015335Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1733408015338 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733408015338Flushing 1588230740/info: creating writer at 1733408015339 (+1 ms)Flushing 1588230740/info: appending metadata at 1733408015388 (+49 ms)Flushing 1588230740/info: closing flushed file at 1733408015391 (+3 ms)Flushing 1588230740/ns: creating writer at 1733408015462 (+71 ms)Flushing 1588230740/ns: appending metadata at 1733408015476 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733408015476Flushing 1588230740/table: creating writer at 1733408015495 (+19 ms)Flushing 1588230740/table: appending metadata at 1733408015508 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733408015508Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44490d6f: reopening flushed file at 1733408015530 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c6739f5: reopening flushed file at 1733408015541 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e93a597: reopening flushed file at 1733408015553 (+12 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 232ms, sequenceid=64, compaction requested=false; wal=null at 1733408015567 (+14 ms)Cleaning up temporary data from old regions at 1733408015572 (+5 ms)Running coprocessor post-open hooks at 1733408015580 (+8 ms)Region opened successfully at 1733408015580 2024-12-05T14:13:35,582 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1733408015273 2024-12-05T14:13:35,584 DEBUG [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T14:13:35,585 INFO [RS_OPEN_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T14:13:35,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:35,586 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:35,624 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T14:13:35,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:35,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:35,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:35,635 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:35,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:35,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:35,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:35,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-05T14:13:35,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 in 515 msec 2024-12-05T14:13:35,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-12-05T14:13:35,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 700 msec 2024-12-05T14:13:35,640 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 218194e0d8ec,40363,1733408008532 had 9 regions 2024-12-05T14:13:35,641 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true, isMeta: false 2024-12-05T14:13:35,643 INFO [PEWorker-1 {}] master.SplitWALManager(105): 218194e0d8ec,40363,1733408008532 WAL count=1, meta=false 2024-12-05T14:13:35,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132}] 2024-12-05T14:13:35,646 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:35,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:35,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36017 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-12-05T14:13:35,817 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132, size=2.7 K (2774bytes) 2024-12-05T14:13:35,817 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132 2024-12-05T14:13:35,818 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132 after 1ms 2024-12-05T14:13:35,821 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132 took 4ms 2024-12-05T14:13:35,824 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for cd727192dbe84371a93aad39b906dd3b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,826 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for abd1856616590a0391ff54d53cfbbb53: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,828 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 894d0c86da04798f8fea0640c4aa6d05: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,830 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a24b719bda0e807d3ebf3858def7531b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,832 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 79e2982faca0d2750624de91bb80c08e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,834 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 88e959e3bb59b49f279be7426309e052: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,836 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for aba2432d09baa11b4c1ea7026329d127: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,837 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 811fb81e434a8467b09b0f283ae0207b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,838 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for c129ce5c0755b4946bdefc2d36f50149: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T14:13:35,839 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132 so closing down 2024-12-05T14:13:35,839 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T14:13:35,839 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T14:13:35,839 INFO [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-12-05T14:13:35,839 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132, journal: Splitting hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132, size=2.7 K (2774bytes) at 1733408015817Finishing writing output for hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132 so closing down at 1733408015839 (+22 ms)3 split writer threads finished at 1733408015839Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1733408015839 2024-12-05T14:13:35,839 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132 2024-12-05T14:13:35,839 DEBUG [RS_LOG_REPLAY_OPS-regionserver/218194e0d8ec:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-05T14:13:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38561 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-05T14:13:35,843 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting/218194e0d8ec%2C40363%2C1733408008532.1733408010132 to hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/oldWALs 2024-12-05T14:13:35,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-05T14:13:35,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 in 196 msec 2024-12-05T14:13:35,847 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:35,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-12-05T14:13:35,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 in 203 msec 2024-12-05T14:13:35,851 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/WALs/218194e0d8ec,40363,1733408008532-splitting dir is empty, no logs to split. 2024-12-05T14:13:35,851 INFO [PEWorker-1 {}] master.SplitWALManager(105): 218194e0d8ec,40363,1733408008532 WAL count=0, meta=false 2024-12-05T14:13:35,851 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 218194e0d8ec,40363,1733408008532 WAL splitting is done? wals=0, meta=false 2024-12-05T14:13:35,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN}] 2024-12-05T14:13:35,855 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:35,856 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:35,857 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:35,858 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:35,859 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:35,860 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:35,861 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:35,862 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:35,863 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:35,864 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:35,865 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:35,866 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:35,867 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:35,868 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:35,869 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:35,870 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T14:13:36,007 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(204): Hosts are {218194e0d8ec=0} racks are {/default-rack=0} 2024-12-05T14:13:36,008 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T14:13:36,008 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T14:13:36,008 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T14:13:36,008 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T14:13:36,008 INFO [218194e0d8ec:38561 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T14:13:36,008 INFO [218194e0d8ec:38561 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T14:13:36,008 DEBUG [218194e0d8ec:38561 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T14:13:36,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=88e959e3bb59b49f279be7426309e052, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,012 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,012 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 218194e0d8ec:40363 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T14:13:36,014 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, regionLocation=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,015 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1, error=java.net.ConnectException: Call to address=218194e0d8ec:40363 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-05T14:13:36,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 from cache 2024-12-05T14:13:36,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, regionLocation=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,017 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T14:13:36,017 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,36017,1733408008679, seqNum=-1] 2024-12-05T14:13:36,018 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T14:13:36,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 218194e0d8ec:40363 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,019 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 218194e0d8ec:40363 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T14:13:36,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,36017,1733408008679, seqNum=-1, error=java.net.ConnectException: Call to address=218194e0d8ec:40363 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,36017,1733408008679, seqNum=-1, error=java.net.ConnectException: Call to address=218194e0d8ec:40363 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,36017,1733408008679, seqNum=-1, error=java.net.ConnectException: Call to address=218194e0d8ec:40363 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,36017,1733408008679, seqNum=-1, error=java.net.ConnectException: Call to address=218194e0d8ec:40363 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,36017,1733408008679, seqNum=-1, error=java.net.ConnectException: Call to address=218194e0d8ec:40363 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,40363,1733408008532, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=218194e0d8ec,36017,1733408008679, seqNum=-1, error=java.net.ConnectException: Call to address=218194e0d8ec:40363 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 218194e0d8ec:40363 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 218194e0d8ec/172.17.0.2:40363 2024-12-05T14:13:36,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49325, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T14:13:36,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN because future has completed 2024-12-05T14:13:36,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:36,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN because future has completed 2024-12-05T14:13:36,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:36,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN because future has completed 2024-12-05T14:13:36,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN because future has completed 2024-12-05T14:13:36,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN because future has completed 2024-12-05T14:13:36,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN because future has completed 2024-12-05T14:13:36,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:36,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN because future has completed 2024-12-05T14:13:36,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN because future has completed 2024-12-05T14:13:36,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:36,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:36,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:36,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,36017,1733408008679}] 2024-12-05T14:13:36,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba2432d09baa11b4c1ea7026329d127, server=218194e0d8ec,45449,1733408008631}] 2024-12-05T14:13:36,176 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T14:13:36,179 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:36,180 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => c129ce5c0755b4946bdefc2d36f50149, NAME => 'test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-05T14:13:36,180 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,180 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:36,180 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,180 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,182 INFO [StoreOpener-c129ce5c0755b4946bdefc2d36f50149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,184 INFO [StoreOpener-c129ce5c0755b4946bdefc2d36f50149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c129ce5c0755b4946bdefc2d36f50149 columnFamilyName family 2024-12-05T14:13:36,184 DEBUG [StoreOpener-c129ce5c0755b4946bdefc2d36f50149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:36,185 INFO [StoreOpener-c129ce5c0755b4946bdefc2d36f50149-1 {}] regionserver.HStore(327): Store=c129ce5c0755b4946bdefc2d36f50149/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:36,185 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,186 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,188 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,188 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,189 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,191 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,192 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened c129ce5c0755b4946bdefc2d36f50149; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63050088, jitterRate=-0.060480475425720215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:36,193 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:36,193 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for c129ce5c0755b4946bdefc2d36f50149: Running coprocessor pre-open hook at 1733408016181Writing region info on filesystem at 1733408016181Initializing all the Stores at 1733408016182 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408016182Cleaning up temporary data from old regions at 1733408016189 (+7 ms)Running coprocessor post-open hooks at 1733408016193 (+4 ms)Region opened successfully at 1733408016193 2024-12-05T14:13:36,194 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149., pid=78, masterSystemTime=1733408016175 2024-12-05T14:13:36,196 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:36,196 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:36,196 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:36,196 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => aba2432d09baa11b4c1ea7026329d127, NAME => 'test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-05T14:13:36,197 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,197 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:36,197 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,197 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,199 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-05T14:13:36,199 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,200 INFO [StoreOpener-aba2432d09baa11b4c1ea7026329d127-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,199 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,201 INFO [StoreOpener-aba2432d09baa11b4c1ea7026329d127-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aba2432d09baa11b4c1ea7026329d127 columnFamilyName family 2024-12-05T14:13:36,201 DEBUG [StoreOpener-aba2432d09baa11b4c1ea7026329d127-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:36,202 INFO [StoreOpener-aba2432d09baa11b4c1ea7026329d127-1 {}] regionserver.HStore(327): Store=aba2432d09baa11b4c1ea7026329d127/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:36,202 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,203 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,204 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,205 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,205 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,208 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,209 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened aba2432d09baa11b4c1ea7026329d127; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71276061, jitterRate=0.06209607422351837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:36,209 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:36,209 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for aba2432d09baa11b4c1ea7026329d127: Running coprocessor pre-open hook at 1733408016197Writing region info on filesystem at 1733408016197Initializing all the Stores at 1733408016198 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408016198Cleaning up temporary data from old regions at 1733408016205 (+7 ms)Running coprocessor post-open hooks at 1733408016209 (+4 ms)Region opened successfully at 1733408016209 2024-12-05T14:13:36,210 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127., pid=82, masterSystemTime=1733408016175 2024-12-05T14:13:36,211 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-05T14:13:36,213 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,214 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba2432d09baa11b4c1ea7026329d127, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,214 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:36,214 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:36,214 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:36,215 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 894d0c86da04798f8fea0640c4aa6d05, NAME => 'test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-05T14:13:36,215 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,215 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:36,215 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba2432d09baa11b4c1ea7026329d127, server=218194e0d8ec,45449,1733408008631 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,215 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,215 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,217 INFO [StoreOpener-894d0c86da04798f8fea0640c4aa6d05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,218 INFO [StoreOpener-894d0c86da04798f8fea0640c4aa6d05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 894d0c86da04798f8fea0640c4aa6d05 columnFamilyName family 2024-12-05T14:13:36,218 DEBUG [StoreOpener-894d0c86da04798f8fea0640c4aa6d05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:36,218 INFO [StoreOpener-894d0c86da04798f8fea0640c4aa6d05-1 {}] regionserver.HStore(327): Store=894d0c86da04798f8fea0640c4aa6d05/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:36,219 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,219 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,221 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,221 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,221 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,223 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,224 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 894d0c86da04798f8fea0640c4aa6d05; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63523088, jitterRate=-0.05343222618103027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:36,224 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:36,225 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 894d0c86da04798f8fea0640c4aa6d05: Running coprocessor pre-open hook at 1733408016215Writing region info on filesystem at 1733408016215Initializing all the Stores at 1733408016216 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408016216Cleaning up temporary data from old regions at 1733408016221 (+5 ms)Running coprocessor post-open hooks at 1733408016225 (+4 ms)Region opened successfully at 1733408016225 2024-12-05T14:13:36,226 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05., pid=75, masterSystemTime=1733408016175 2024-12-05T14:13:36,228 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:36,228 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:36,228 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:36,229 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => cd727192dbe84371a93aad39b906dd3b, NAME => 'test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-05T14:13:36,229 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,229 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:36,229 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,229 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,231 INFO [StoreOpener-cd727192dbe84371a93aad39b906dd3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,232 INFO [StoreOpener-cd727192dbe84371a93aad39b906dd3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd727192dbe84371a93aad39b906dd3b columnFamilyName family 2024-12-05T14:13:36,232 DEBUG [StoreOpener-cd727192dbe84371a93aad39b906dd3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:36,233 INFO [StoreOpener-cd727192dbe84371a93aad39b906dd3b-1 {}] regionserver.HStore(327): Store=cd727192dbe84371a93aad39b906dd3b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:36,233 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,234 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,235 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-05T14:13:36,236 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,236 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,236 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,238 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,239 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened cd727192dbe84371a93aad39b906dd3b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63759197, jitterRate=-0.0499139279127121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:36,240 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:36,240 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for cd727192dbe84371a93aad39b906dd3b: Running coprocessor pre-open hook at 1733408016229Writing region info on filesystem at 1733408016229Initializing all the Stores at 1733408016230 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408016231 (+1 ms)Cleaning up temporary data from old regions at 1733408016237 (+6 ms)Running coprocessor post-open hooks at 1733408016240 (+3 ms)Region opened successfully at 1733408016240 2024-12-05T14:13:36,241 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b., pid=79, masterSystemTime=1733408016175 2024-12-05T14:13:36,242 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,242 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,243 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,243 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:36,243 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:36,243 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,244 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,244 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,244 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,252 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,253 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,254 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba2432d09baa11b4c1ea7026329d127, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,266 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,266 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba2432d09baa11b4c1ea7026329d127, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,267 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-12-05T14:13:36,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,269 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,270 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,289 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:36,289 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => a24b719bda0e807d3ebf3858def7531b, NAME => 'test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-05T14:13:36,289 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,289 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:36,290 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,290 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,291 INFO [StoreOpener-a24b719bda0e807d3ebf3858def7531b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,292 INFO [StoreOpener-a24b719bda0e807d3ebf3858def7531b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a24b719bda0e807d3ebf3858def7531b columnFamilyName family 2024-12-05T14:13:36,292 DEBUG [StoreOpener-a24b719bda0e807d3ebf3858def7531b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:36,293 INFO [StoreOpener-a24b719bda0e807d3ebf3858def7531b-1 {}] regionserver.HStore(327): Store=a24b719bda0e807d3ebf3858def7531b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:36,293 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,294 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,295 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,296 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,296 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,298 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,299 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened a24b719bda0e807d3ebf3858def7531b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64706046, jitterRate=-0.03580477833747864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:36,299 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:36,299 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for a24b719bda0e807d3ebf3858def7531b: Running coprocessor pre-open hook at 1733408016290Writing region info on filesystem at 1733408016290Initializing all the Stores at 1733408016291 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408016291Cleaning up temporary data from old regions at 1733408016296 (+5 ms)Running coprocessor post-open hooks at 1733408016299 (+3 ms)Region opened successfully at 1733408016299 2024-12-05T14:13:36,300 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b., pid=76, masterSystemTime=1733408016284 2024-12-05T14:13:36,301 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,318 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "218194e0d8ec" port: 36017 start_code: 1733408008679 } transition { transition_code: OPENED region_info { region_id: 1733408011017 table_name { namespace: "default" qualifier: "test" } start_key: "zzz" end_key: "" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1733408008785 }; retry (#0) after 1008ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-05T14:13:36,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,321 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,321 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,321 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,321 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,321 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,321 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,321 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,321 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,322 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046322 2024-12-05T14:13:36,322 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,322 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,324 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,325 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,325 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,325 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,354 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,354 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076354 2024-12-05T14:13:36,354 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136354 2024-12-05T14:13:36,354 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-12-05T14:13:36,354 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,355 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,355 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,355 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,355 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,355 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,355 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,355 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,356 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,360 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,361 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,367 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,367 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88e959e3bb59b49f279be7426309e052, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,368 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-12-05T14:13:36,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,369 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,369 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,384 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,384 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,384 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,384 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,384 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,384 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,384 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046384 2024-12-05T14:13:36,384 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,384 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,386 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,387 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,387 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,387 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,401 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,401 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,401 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076401 2024-12-05T14:13:36,401 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136401 2024-12-05T14:13:36,401 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-12-05T14:13:36,401 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,401 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,402 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,402 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,402 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,402 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,403 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,416 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 811fb81e434a8467b09b0f283ae0207b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,418 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-12-05T14:13:36,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,419 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,420 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,432 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,432 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046432 2024-12-05T14:13:36,432 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,432 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,434 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,436 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,436 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,436 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,447 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,448 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076447 2024-12-05T14:13:36,448 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136448 2024-12-05T14:13:36,448 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-05T14:13:36,448 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,448 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,448 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,448 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,448 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,448 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,449 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,449 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,450 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd727192dbe84371a93aad39b906dd3b, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,463 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-12-05T14:13:36,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,464 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,464 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,478 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,478 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,479 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,479 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,479 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,479 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,479 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,479 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,479 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046479 2024-12-05T14:13:36,479 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,479 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,481 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,483 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,484 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,484 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,493 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,493 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076493 2024-12-05T14:13:36,493 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136493 2024-12-05T14:13:36,493 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-12-05T14:13:36,493 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,493 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,493 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,493 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,493 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,493 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,494 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,494 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,498 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,499 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,504 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,504 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,504 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,505 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,505 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,505 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,505 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,505 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,505 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,505 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-12-05T14:13:36,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,506 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,507 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,521 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046521 2024-12-05T14:13:36,521 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,521 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,522 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,523 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,523 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,524 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,530 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,530 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076530 2024-12-05T14:13:36,530 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136530 2024-12-05T14:13:36,530 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T14:13:36,531 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,531 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,531 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,531 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,531 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,531 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,531 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,531 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,532 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,535 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,539 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,539 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,539 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,539 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,539 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,539 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,539 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd1856616590a0391ff54d53cfbbb53, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,539 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-05T14:13:36,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,540 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,540 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,542 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,547 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,552 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,552 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046552 2024-12-05T14:13:36,552 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,552 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,553 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,554 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,554 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,554 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,561 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,562 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076561 2024-12-05T14:13:36,562 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136562 2024-12-05T14:13:36,562 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T14:13:36,562 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,562 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,562 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,562 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,562 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,563 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,563 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,565 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,568 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,569 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24b719bda0e807d3ebf3858def7531b, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T14:13:36,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,570 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,570 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,584 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046584 2024-12-05T14:13:36,584 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,584 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,585 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,586 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,586 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,587 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,591 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,591 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076591 2024-12-05T14:13:36,591 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136591 2024-12-05T14:13:36,591 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T14:13:36,591 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,591 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,591 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,591 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,591 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,592 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,598 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,598 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,598 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,598 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,598 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,599 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,599 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,599 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,599 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,599 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,599 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 894d0c86da04798f8fea0640c4aa6d05, server=218194e0d8ec,45449,1733408008631 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-05T14:13:36,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,600 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,600 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,610 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046610 2024-12-05T14:13:36,611 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,611 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,614 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,614 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,614 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,618 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,618 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076618 2024-12-05T14:13:36,618 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136618 2024-12-05T14:13:36,619 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-05T14:13:36,619 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,619 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,619 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,619 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,619 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,619 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,619 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,619 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,620 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,624 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,624 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,624 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,624 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,624 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN 2024-12-05T14:13:36,625 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T14:13:36,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,626 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,626 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=811fb81e434a8467b09b0f283ae0207b, ASSIGN to state=OFFLINE, location=null, table=test, region=811fb81e434a8467b09b0f283ae0207b to restore RIT 2024-12-05T14:13:36,637 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046636 2024-12-05T14:13:36,637 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,637 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,638 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,639 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,639 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,640 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,645 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,645 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076645 2024-12-05T14:13:36,645 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136645 2024-12-05T14:13:36,645 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-05T14:13:36,645 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,645 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,645 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,645 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,646 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,646 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,646 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,647 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,649 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,653 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,652 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,653 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,653 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,653 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN 2024-12-05T14:13:36,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-05T14:13:36,653 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,654 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,654 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,667 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cd727192dbe84371a93aad39b906dd3b, ASSIGN to state=OFFLINE, location=null, table=test, region=cd727192dbe84371a93aad39b906dd3b to restore RIT 2024-12-05T14:13:36,668 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046668 2024-12-05T14:13:36,668 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,668 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,671 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,673 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,673 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,673 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,676 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,676 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076676 2024-12-05T14:13:36,676 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136676 2024-12-05T14:13:36,676 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T14:13:36,676 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,676 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,676 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,676 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,676 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,676 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,676 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,676 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,677 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,680 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,680 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,681 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,682 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,682 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,682 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,682 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,682 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,682 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,682 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN 2024-12-05T14:13:36,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T14:13:36,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,683 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,683 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c129ce5c0755b4946bdefc2d36f50149, ASSIGN to state=OFFLINE, location=null, table=test, region=c129ce5c0755b4946bdefc2d36f50149 to restore RIT 2024-12-05T14:13:36,695 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046695 2024-12-05T14:13:36,695 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,695 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,696 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,697 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,697 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,697 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076704 2024-12-05T14:13:36,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136704 2024-12-05T14:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T14:13:36,704 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,704 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,705 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,705 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,705 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,705 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,705 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,712 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,712 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN 2024-12-05T14:13:36,712 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-05T14:13:36,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,713 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,713 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,725 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,726 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,726 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,726 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,726 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=894d0c86da04798f8fea0640c4aa6d05, ASSIGN to state=OFFLINE, location=null, table=test, region=894d0c86da04798f8fea0640c4aa6d05 to restore RIT 2024-12-05T14:13:36,726 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046726 2024-12-05T14:13:36,726 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,726 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,727 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,729 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,729 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,729 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,736 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,736 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076736 2024-12-05T14:13:36,736 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136736 2024-12-05T14:13:36,737 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-05T14:13:36,737 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,737 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,737 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,737 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,737 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,737 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,737 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,737 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,738 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,740 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,740 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,740 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,740 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,742 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,742 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,742 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,742 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,742 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN 2024-12-05T14:13:36,742 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T14:13:36,743 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,743 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,743 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,751 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,751 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,751 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,751 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,752 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,752 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,752 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88e959e3bb59b49f279be7426309e052, ASSIGN to state=OFFLINE, location=null, table=test, region=88e959e3bb59b49f279be7426309e052 to restore RIT 2024-12-05T14:13:36,752 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046752 2024-12-05T14:13:36,752 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,752 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,753 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,754 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,754 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,754 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,761 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,761 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076761 2024-12-05T14:13:36,761 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136761 2024-12-05T14:13:36,762 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T14:13:36,762 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,762 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,762 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,762 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,762 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,762 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,762 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,763 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,763 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,768 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,768 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,768 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,768 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN 2024-12-05T14:13:36,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T14:13:36,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,769 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,770 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,782 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,782 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,782 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,783 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,783 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,783 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=aba2432d09baa11b4c1ea7026329d127, ASSIGN to state=OFFLINE, location=null, table=test, region=aba2432d09baa11b4c1ea7026329d127 to restore RIT 2024-12-05T14:13:36,784 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046783 2024-12-05T14:13:36,784 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,784 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,785 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,786 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,786 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,786 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,791 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,791 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076791 2024-12-05T14:13:36,791 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136791 2024-12-05T14:13:36,791 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T14:13:36,792 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,792 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,792 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,792 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,792 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,792 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,792 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,792 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,794 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,798 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN 2024-12-05T14:13:36,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T14:13:36,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,799 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,799 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,858 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,858 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,871 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,871 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a24b719bda0e807d3ebf3858def7531b, ASSIGN to state=OFFLINE, location=null, table=test, region=a24b719bda0e807d3ebf3858def7531b to restore RIT 2024-12-05T14:13:36,871 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046871 2024-12-05T14:13:36,871 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,871 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,873 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,874 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,874 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,874 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,879 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,879 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076879 2024-12-05T14:13:36,879 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136879 2024-12-05T14:13:36,879 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T14:13:36,879 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,879 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,879 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,879 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,880 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,880 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,880 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,880 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,881 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN 2024-12-05T14:13:36,884 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-05T14:13:36,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,885 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,885 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:36,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:36,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:36,958 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abd1856616590a0391ff54d53cfbbb53, ASSIGN to state=OFFLINE, location=null, table=test, region=abd1856616590a0391ff54d53cfbbb53 to restore RIT 2024-12-05T14:13:36,960 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408046960 2024-12-05T14:13:36,960 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:36,960 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,962 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:36,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:36,963 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:36,971 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:36,971 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408076971 2024-12-05T14:13:36,971 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408136971 2024-12-05T14:13:36,971 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-05T14:13:36,972 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:36,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:36,972 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:36,972 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:36,972 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:36,972 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:36,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:36,972 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,972 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:36,973 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:36,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:36,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:36,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:36,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:36,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T14:13:36,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:36,978 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:36,978 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:37,078 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,079 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,155 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408047155 2024-12-05T14:13:37,155 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:37,155 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,156 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:37,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:37,158 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,163 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:37,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408077163 2024-12-05T14:13:37,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408137163 2024-12-05T14:13:37,163 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T14:13:37,163 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:37,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,163 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:37,163 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:37,163 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:37,163 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:37,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:37,163 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:37,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:37,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:37,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:37,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:37,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:37,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:37,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:37,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:37,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:37,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:37,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:37,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:37,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.1733408010132, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,167 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:37,167 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:37,168 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:37,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,189 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408047189 2024-12-05T14:13:37,189 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:37,189 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:37,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:37,191 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,194 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:37,194 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408077194 2024-12-05T14:13:37,194 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408137194 2024-12-05T14:13:37,194 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-05T14:13:37,194 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:37,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,194 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:37,194 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:37,194 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:37,195 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:37,195 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:37,195 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:37,195 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:37,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:37,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:37,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:37,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:37,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T14:13:37,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:37,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:37,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,198 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-05T14:13:37,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:37,199 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:37,199 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:37,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,210 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408047210 2024-12-05T14:13:37,210 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:37,210 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:37,213 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:37,213 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:37,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,213 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,218 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:37,218 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408077218 2024-12-05T14:13:37,218 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408137218 2024-12-05T14:13:37,218 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T14:13:37,218 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:37,218 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,218 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:37,218 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:37,218 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:37,218 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:37,218 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:37,218 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:37,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:37,220 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:37,221 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:37,221 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:37,221 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T14:13:37,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:37,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:37,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:37,222 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:37,222 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,222 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,222 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T14:13:37,222 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:37,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-05T14:13:37,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:37,223 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:37,223 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:37,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,236 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408047236 2024-12-05T14:13:37,236 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:37,236 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,237 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:37,238 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:37,238 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,238 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:37,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,242 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:37,242 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408077242 2024-12-05T14:13:37,242 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408137242 2024-12-05T14:13:37,242 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-05T14:13:37,242 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:37,242 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,242 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:37,242 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:37,242 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:37,242 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:37,242 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:37,243 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:37,243 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:37,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:37,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:37,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T14:13:37,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:37,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:37,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:37,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:37,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,246 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:37,246 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,246 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,246 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-05T14:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:37,246 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:37,247 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:37,255 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,255 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,257 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408047257 2024-12-05T14:13:37,257 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:37,257 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,258 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:37,259 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:37,259 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,259 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:37,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,265 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:37,265 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408077265 2024-12-05T14:13:37,265 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408137265 2024-12-05T14:13:37,265 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-05T14:13:37,265 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:37,265 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,265 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:37,265 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:37,266 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:37,266 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:37,266 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:37,266 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:37,267 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:37,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:37,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T14:13:37,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:37,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:37,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:37,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:37,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,270 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:37,270 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 218194e0d8ec%2C40363%2C1733408008532.meta.1733408010633.meta, worker=218194e0d8ec,36017,1733408008679 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T14:13:37,270 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-05T14:13:37,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:37,271 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:37,271 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:37,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,283 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408047283 2024-12-05T14:13:37,283 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:37,284 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,285 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:37,286 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T14:13:37,286 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:37,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,286 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,293 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:37,293 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408077293 2024-12-05T14:13:37,293 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408137293 2024-12-05T14:13:37,293 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T14:13:37,293 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T14:13:37,293 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,293 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:37,293 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:37,293 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T14:13:37,293 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T14:13:37,293 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T14:13:37,294 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T14:13:37,294 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T14:13:37,296 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T14:13:37,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T14:13:37,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T14:13:37,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79e2982faca0d2750624de91bb80c08e, REOPEN/MOVE 2024-12-05T14:13:37,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true 2024-12-05T14:13:37,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T14:13:37,297 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-05T14:13:37,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=65 2024-12-05T14:13:37,298 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=218194e0d8ec,36017,1733408008679, table=hbase:meta, region=1588230740 2024-12-05T14:13:37,298 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 218194e0d8ec,36017,1733408008679, state=OPEN 2024-12-05T14:13:37,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T14:13:37,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T14:13:37,310 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733408047310 2024-12-05T14:13:37,310 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T14:13:37,310 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T14:13:37,311 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T14:13:37,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c129ce5c0755b4946bdefc2d36f50149, server=218194e0d8ec,45449,1733408008631:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 218194e0d8ec,40363,1733408008532, splitWal=true, meta=true exec-time=2.6640 sec 2024-12-05T14:13:37,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88bd874c8e3a659802f7bbb662036a12, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=27fe33c5bc228b6793a2d277d2d7d9c2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=894d0c86da04798f8fea0640c4aa6d05, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ffe5657e6841580f09ad342b0a1a3ad, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c703cd272011369700660d18171711b, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abd1856616590a0391ff54d53cfbbb53, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aba2432d09baa11b4c1ea7026329d127, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59e732700952e1f67c04f79b391abf8c, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3fe46a05dda7cc73743c5e741892f632, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b645f799e60d0a296066576eac0b0285, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=811fb81e434a8467b09b0f283ae0207b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4edea3efe3103a410142665ac1ea3d83, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffa1b4ffdfeddf082998369feea2e28b, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c129ce5c0755b4946bdefc2d36f50149, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dd338d2cdec8bcc235bdb212f336d28, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce07d807a24e01c939692ade9b01d97e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88e959e3bb59b49f279be7426309e052, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44e3fd15a9de4a8ceefe8e8505a163f2, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=70c6030455907dcfb74337bc850642ff, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd727192dbe84371a93aad39b906dd3b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=564487ebab3bd815a85a50262b275a91, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c14601cce05aed77304a5058eff93ba1, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b620ca010e58c2eb2f2309f4a11c2dba, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=2 2024-12-05T14:13:37,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79e2982faca0d2750624de91bb80c08e, regionState=OPEN, lastHost=218194e0d8ec,45449,1733408008631, regionLocation=218194e0d8ec,45449,1733408008631, openSeqNum=5 2024-12-05T14:13:37,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=710e86e039ffc8d96f214260888d4849, regionState=OPEN, lastHost=218194e0d8ec,36017,1733408008679, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24b719bda0e807d3ebf3858def7531b, regionState=OPENING, lastHost=218194e0d8ec,40363,1733408008532, regionLocation=218194e0d8ec,36017,1733408008679, openSeqNum=2 2024-12-05T14:13:37,315 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T14:13:37,315 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733408077315 2024-12-05T14:13:37,315 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733408137315 2024-12-05T14:13:37,315 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-12-05T14:13:37,328 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1316): No matching procedure found for 218194e0d8ec,36017,1733408008679 transition on state=OPENING, location=218194e0d8ec,36017,1733408008679, table=test, region=a24b719bda0e807d3ebf3858def7531b to OPENED 2024-12-05T14:13:37,329 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2353): TRANSITION REPORTED server { host_name: "218194e0d8ec" port: 36017 start_code: 1733408008679 } transition { transition_code: OPENED region_info { region_id: 1733408011017 table_name { namespace: "default" qualifier: "test" } start_key: "zzz" end_key: "" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1733408008785 } 2024-12-05T14:13:37,329 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:37,329 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:37,329 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:37,329 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 811fb81e434a8467b09b0f283ae0207b, NAME => 'test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-05T14:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,331 INFO [StoreOpener-811fb81e434a8467b09b0f283ae0207b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,332 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 382), OpenFileDescriptor=622 (was 645), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=392 (was 392), ProcessCount=11 (was 11), AvailableMemoryMB=6242 (was 6289) 2024-12-05T14:13:37,332 INFO [StoreOpener-811fb81e434a8467b09b0f283ae0207b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 811fb81e434a8467b09b0f283ae0207b columnFamilyName family 2024-12-05T14:13:37,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T14:13:37,332 DEBUG [StoreOpener-811fb81e434a8467b09b0f283ae0207b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:37,332 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T14:13:37,332 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T14:13:37,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:37,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:37,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T14:13:37,333 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T14:13:37,333 INFO [StoreOpener-811fb81e434a8467b09b0f283ae0207b-1 {}] regionserver.HStore(327): Store=811fb81e434a8467b09b0f283ae0207b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:37,333 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=563373474, stopped=false 2024-12-05T14:13:37,333 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,334 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=218194e0d8ec,38561,1733408007726 2024-12-05T14:13:37,334 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,336 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,336 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,336 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,338 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,339 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 811fb81e434a8467b09b0f283ae0207b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59968734, jitterRate=-0.10639622807502747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:37,339 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,339 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 811fb81e434a8467b09b0f283ae0207b: Running coprocessor pre-open hook at 1733408017330Writing region info on filesystem at 1733408017330Initializing all the Stores at 1733408017331 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408017331Cleaning up temporary data from old regions at 1733408017336 (+5 ms)Running coprocessor post-open hooks at 1733408017339 (+3 ms)Region opened successfully at 1733408017339 2024-12-05T14:13:37,340 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b., pid=80, masterSystemTime=1733408016284 2024-12-05T14:13:37,341 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38561 {}] assignment.AssignmentManager(1316): No matching procedure found for 218194e0d8ec,36017,1733408008679 transition on state=OPENING, location=218194e0d8ec,36017,1733408008679, table=test, region=811fb81e434a8467b09b0f283ae0207b to OPENED 2024-12-05T14:13:37,341 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:37,341 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:37,341 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. 2024-12-05T14:13:37,341 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => abd1856616590a0391ff54d53cfbbb53, NAME => 'test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-05T14:13:37,342 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,342 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:37,342 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,342 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,343 INFO [StoreOpener-abd1856616590a0391ff54d53cfbbb53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,344 INFO [StoreOpener-abd1856616590a0391ff54d53cfbbb53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd1856616590a0391ff54d53cfbbb53 columnFamilyName family 2024-12-05T14:13:37,344 DEBUG [StoreOpener-abd1856616590a0391ff54d53cfbbb53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:37,345 INFO [StoreOpener-abd1856616590a0391ff54d53cfbbb53-1 {}] regionserver.HStore(327): Store=abd1856616590a0391ff54d53cfbbb53/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:37,345 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T14:13:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T14:13:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T14:13:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:37,346 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:37,346 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T14:13:37,346 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,346 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T14:13:37,346 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:13:37,346 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:13:37,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:37,346 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T14:13:37,347 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '218194e0d8ec,45449,1733408008631' ***** 2024-12-05T14:13:37,347 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T14:13:37,347 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '218194e0d8ec,36017,1733408008679' ***** 2024-12-05T14:13:37,347 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T14:13:37,347 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T14:13:37,347 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T14:13:37,347 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T14:13:37,347 INFO [RS:1;218194e0d8ec:45449 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T14:13:37,347 INFO [RS:2;218194e0d8ec:36017 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T14:13:37,347 INFO [RS:1;218194e0d8ec:45449 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T14:13:37,347 INFO [RS:2;218194e0d8ec:36017 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T14:13:37,347 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for b645f799e60d0a296066576eac0b0285 2024-12-05T14:13:37,347 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for 3fe46a05dda7cc73743c5e741892f632 2024-12-05T14:13:37,348 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ffe5657e6841580f09ad342b0a1a3ad 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 9dd338d2cdec8bcc235bdb212f336d28 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 710e86e039ffc8d96f214260888d4849 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for 27fe33c5bc228b6793a2d277d2d7d9c2 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 811fb81e434a8467b09b0f283ae0207b 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for 44e3fd15a9de4a8ceefe8e8505a163f2 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for 79e2982faca0d2750624de91bb80c08e 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 70c6030455907dcfb74337bc850642ff 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for aba2432d09baa11b4c1ea7026329d127 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for 4edea3efe3103a410142665ac1ea3d83 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 88bd874c8e3a659802f7bbb662036a12 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for cd727192dbe84371a93aad39b906dd3b 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for c14601cce05aed77304a5058eff93ba1 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for 564487ebab3bd815a85a50262b275a91 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for a24b719bda0e807d3ebf3858def7531b 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for 5c703cd272011369700660d18171711b 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 59e732700952e1f67c04f79b391abf8c 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for b620ca010e58c2eb2f2309f4a11c2dba 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for c129ce5c0755b4946bdefc2d36f50149 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(3091): Received CLOSE for 894d0c86da04798f8fea0640c4aa6d05 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(959): stopping server 218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(959): stopping server 218194e0d8ec,45449,1733408008631 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T14:13:37,348 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b645f799e60d0a296066576eac0b0285, disabling compactions & flushes 2024-12-05T14:13:37,348 INFO [RS:1;218194e0d8ec:45449 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;218194e0d8ec:45449. 2024-12-05T14:13:37,348 INFO [RS:2;218194e0d8ec:36017 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;218194e0d8ec:36017. 2024-12-05T14:13:37,348 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:37,349 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:37,349 DEBUG [RS:1;218194e0d8ec:45449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T14:13:37,349 DEBUG [RS:2;218194e0d8ec:36017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T14:13:37,349 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. after waiting 0 ms 2024-12-05T14:13:37,349 DEBUG [RS:1;218194e0d8ec:45449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:37,349 DEBUG [RS:2;218194e0d8ec:36017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:37,349 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:37,349 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-12-05T14:13:37,349 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T14:13:37,349 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T14:13:37,349 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T14:13:37,349 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T14:13:37,349 DEBUG [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(1351): Waiting on 27fe33c5bc228b6793a2d277d2d7d9c2, 3fe46a05dda7cc73743c5e741892f632, 44e3fd15a9de4a8ceefe8e8505a163f2, 4edea3efe3103a410142665ac1ea3d83, 564487ebab3bd815a85a50262b275a91, 5c703cd272011369700660d18171711b, 79e2982faca0d2750624de91bb80c08e, 894d0c86da04798f8fea0640c4aa6d05, aba2432d09baa11b4c1ea7026329d127, b620ca010e58c2eb2f2309f4a11c2dba, c129ce5c0755b4946bdefc2d36f50149, cd727192dbe84371a93aad39b906dd3b, ce07d807a24e01c939692ade9b01d97e 2024-12-05T14:13:37,349 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,349 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-12-05T14:13:37,349 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3fe46a05dda7cc73743c5e741892f632, disabling compactions & flushes 2024-12-05T14:13:37,350 DEBUG [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(1351): Waiting on 0ffe5657e6841580f09ad342b0a1a3ad, 1588230740, 59e732700952e1f67c04f79b391abf8c, 70c6030455907dcfb74337bc850642ff, 710e86e039ffc8d96f214260888d4849, 811fb81e434a8467b09b0f283ae0207b, 88bd874c8e3a659802f7bbb662036a12, 9dd338d2cdec8bcc235bdb212f336d28, a24b719bda0e807d3ebf3858def7531b, abd1856616590a0391ff54d53cfbbb53, b645f799e60d0a296066576eac0b0285, c14601cce05aed77304a5058eff93ba1, ffa1b4ffdfeddf082998369feea2e28b 2024-12-05T14:13:37,350 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:37,350 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T14:13:37,350 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:37,350 INFO [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T14:13:37,350 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. after waiting 0 ms 2024-12-05T14:13:37,350 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T14:13:37,350 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:37,350 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T14:13:37,350 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T14:13:37,350 INFO [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-12-05T14:13:37,350 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,350 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,353 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,354 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/b645f799e60d0a296066576eac0b0285/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,354 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened abd1856616590a0391ff54d53cfbbb53; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70580790, jitterRate=0.05173572897911072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:37,354 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abd1856616590a0391ff54d53cfbbb53 2024-12-05T14:13:37,354 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for abd1856616590a0391ff54d53cfbbb53: Running coprocessor pre-open hook at 1733408017342Writing region info on filesystem at 1733408017342Initializing all the Stores at 1733408017343 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408017343Cleaning up temporary data from old regions at 1733408017350 (+7 ms)Running coprocessor post-open hooks at 1733408017354 (+4 ms)Region opened successfully at 1733408017354 2024-12-05T14:13:37,355 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:37,355 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b645f799e60d0a296066576eac0b0285: Waiting for close lock at 1733408017348Running coprocessor pre-close hooks at 1733408017348Disabling compacts and flushes for region at 1733408017348Disabling writes for close at 1733408017349 (+1 ms)Writing region close event to WAL at 1733408017349Running coprocessor post-close hooks at 1733408017355 (+6 ms)Closed at 1733408017355 2024-12-05T14:13:37,355 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733408011017.b645f799e60d0a296066576eac0b0285. 2024-12-05T14:13:37,356 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/3fe46a05dda7cc73743c5e741892f632/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,356 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ffe5657e6841580f09ad342b0a1a3ad, disabling compactions & flushes 2024-12-05T14:13:37,356 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:37,356 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:37,356 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. after waiting 0 ms 2024-12-05T14:13:37,356 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:37,357 WARN [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53., aborting... org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server 218194e0d8ec,36017,1733408008679 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1466) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T14:13:37,357 ERROR [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 218194e0d8ec,36017,1733408008679: Failed to open region test,fff,1733408011017.abd1856616590a0391ff54d53cfbbb53. and can not recover ***** org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server 218194e0d8ec,36017,1733408008679 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1466) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T14:13:37,358 ERROR [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-05T14:13:37,358 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-05T14:13:37,359 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-05T14:13:37,360 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:37,360 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3fe46a05dda7cc73743c5e741892f632: Waiting for close lock at 1733408017349Running coprocessor pre-close hooks at 1733408017349Disabling compacts and flushes for region at 1733408017349Disabling writes for close at 1733408017350 (+1 ms)Writing region close event to WAL at 1733408017350Running coprocessor post-close hooks at 1733408017359 (+9 ms)Closed at 1733408017360 (+1 ms) 2024-12-05T14:13:37,360 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733408011017.3fe46a05dda7cc73743c5e741892f632. 2024-12-05T14:13:37,360 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ce07d807a24e01c939692ade9b01d97e, disabling compactions & flushes 2024-12-05T14:13:37,360 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:37,360 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:37,360 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. after waiting 0 ms 2024-12-05T14:13:37,360 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:37,361 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-05T14:13:37,362 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-05T14:13:37,363 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/0ffe5657e6841580f09ad342b0a1a3ad/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,365 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:37,365 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ffe5657e6841580f09ad342b0a1a3ad: Waiting for close lock at 1733408017356Running coprocessor pre-close hooks at 1733408017356Disabling compacts and flushes for region at 1733408017356Disabling writes for close at 1733408017356Writing region close event to WAL at 1733408017356Running coprocessor post-close hooks at 1733408017365 (+9 ms)Closed at 1733408017365 2024-12-05T14:13:37,365 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733408011017.0ffe5657e6841580f09ad342b0a1a3ad. 2024-12-05T14:13:37,365 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9dd338d2cdec8bcc235bdb212f336d28, disabling compactions & flushes 2024-12-05T14:13:37,365 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:37,365 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:37,365 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. after waiting 0 ms 2024-12-05T14:13:37,365 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:37,367 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/ce07d807a24e01c939692ade9b01d97e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,369 INFO [regionserver/218194e0d8ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T14:13:37,369 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 503065984 }, "NonHeapMemoryUsage": { "committed": 163774464, "init": 7667712, "max": -1, "used": 161194312 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "218194e0d8ec", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 38860, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 114, "ProcessCallTime_min": 0, "ProcessCallTime_max": 30, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 15, "ProcessCallTime_75th_percentile": 22, "ProcessCallTime_90th_percentile": 27, "ProcessCallTime_95th_percentile": 28, "ProcessCallTime_98th_percentile": 29, "ProcessCallTime_99th_percentile": 29, "ProcessCallTime_99.9th_percentile": 29, "ProcessCallTime_TimeRangeCount_0-1": 114, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 114, "QueueCallTime_min": 0, "QueueCallTime_max": 9, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 4, "QueueCallTime_75th_percentile": 6, "QueueCallTime_90th_percentile": 8, "QueueCallTime_95th_percentile": 8, "QueueCallTime_98th_percentile": 8, "QueueCallTime_99th_percentile": 8, "QueueCallTime_99.9th_percentile": 8, "QueueCallTime_TimeRangeCount_0-1": 114, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 114, "TotalCallTime_min": 0, "TotalCallTime_max": 30, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 15, "TotalCallTime_75th_percentile": 22, "TotalCallTime_90th_percentile": 27, "TotalCallTime_95th_percentile": 28, "TotalCallTime_98th_percentile": 29, "TotalCallTime_99th_percentile": 29, "TotalCallTime_99.9th_percentile": 29, "TotalCallTime_TimeRangeCount_0-1": 114, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 114, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 600, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 114, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 114, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 308, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 114, "sentBytes": 54885 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "218194e0d8ec", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62933", "tag.serverName": "218194e0d8ec,36017,1733408008679", "tag.clusterId": "39539233-e7ac-4ffc-b25d-6bb3badb36cc", "tag.Context": "regionserver", "tag.Hostname": "218194e0d8ec", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733408008679, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7456, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 17, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 1, "Put_max": 2, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 1, "Put_75th_percentile": 1, "Put_90th_percentile": 1, "Put_95th_percentile": 1, "Put_98th_percentile": 1, "Put_99th_percentile": 1, "Put_99.9th_percentile": 1, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46694, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46694, "FlushOutputSize_max": 46694, "FlushOutputSize_mean": 46694, "FlushOutputSize_25th_percentile": 46694, "FlushOutputSize_median": 46694, "FlushOutputSize_75th_percentile": 46694, "FlushOutputSize_90th_percentile": 46694, "FlushOutputSize_95th_percentile": 46694, "FlushOutputSize_98th_percentile": 46694, "FlushOutputSize_99th_percentile": 46694, "FlushOutputSize_99.9th_percentile": 46694, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 232, "FlushTime_max": 232, "FlushTime_mean": 232, "FlushTime_25th_percentile": 232, "FlushTime_median": 232, "FlushTime_75th_percentile": 232, "FlushTime_90th_percentile": 232, "FlushTime_95th_percentile": 232, "FlushTime_98th_percentile": 232, "FlushTime_99th_percentile": 232, "FlushTime_99.9th_percentile": 232, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-05T14:13:37,369 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/info/b9130c4d5cdd4f639c4a93a5413bc4ae is 138, key is test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05./info:regioninfo/1733408016017/Put/seqid=0 2024-12-05T14:13:37,370 INFO [regionserver/218194e0d8ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T14:13:37,371 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:37,371 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ce07d807a24e01c939692ade9b01d97e: Waiting for close lock at 1733408017360Running coprocessor pre-close hooks at 1733408017360Disabling compacts and flushes for region at 1733408017360Disabling writes for close at 1733408017360Writing region close event to WAL at 1733408017361 (+1 ms)Running coprocessor post-close hooks at 1733408017371 (+10 ms)Closed at 1733408017371 2024-12-05T14:13:37,371 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1733408011017.ce07d807a24e01c939692ade9b01d97e. 2024-12-05T14:13:37,371 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 27fe33c5bc228b6793a2d277d2d7d9c2, disabling compactions & flushes 2024-12-05T14:13:37,371 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:37,371 WARN [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=218194e0d8ec:38561 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=218194e0d8ec:38561 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-12-05T14:13:37,371 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:37,371 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. after waiting 0 ms 2024-12-05T14:13:37,371 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:37,371 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. 2024-12-05T14:13:37,372 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 88e959e3bb59b49f279be7426309e052, NAME => 'test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-05T14:13:37,372 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,372 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T14:13:37,372 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,372 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,375 INFO [StoreOpener-88e959e3bb59b49f279be7426309e052-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,377 INFO [StoreOpener-88e959e3bb59b49f279be7426309e052-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88e959e3bb59b49f279be7426309e052 columnFamilyName family 2024-12-05T14:13:37,378 DEBUG [StoreOpener-88e959e3bb59b49f279be7426309e052-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T14:13:37,378 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/27fe33c5bc228b6793a2d277d2d7d9c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,378 INFO [StoreOpener-88e959e3bb59b49f279be7426309e052-1 {}] regionserver.HStore(327): Store=88e959e3bb59b49f279be7426309e052/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T14:13:37,378 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,379 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:37,379 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 27fe33c5bc228b6793a2d277d2d7d9c2: Waiting for close lock at 1733408017371Running coprocessor pre-close hooks at 1733408017371Disabling compacts and flushes for region at 1733408017371Disabling writes for close at 1733408017371Writing region close event to WAL at 1733408017372 (+1 ms)Running coprocessor post-close hooks at 1733408017379 (+7 ms)Closed at 1733408017379 2024-12-05T14:13:37,379 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733408011017.27fe33c5bc228b6793a2d277d2d7d9c2. 2024-12-05T14:13:37,379 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 44e3fd15a9de4a8ceefe8e8505a163f2, disabling compactions & flushes 2024-12-05T14:13:37,379 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:37,379 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:37,379 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. after waiting 0 ms 2024-12-05T14:13:37,379 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:37,379 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,380 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/9dd338d2cdec8bcc235bdb212f336d28/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,380 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:37,380 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9dd338d2cdec8bcc235bdb212f336d28: Waiting for close lock at 1733408017365Running coprocessor pre-close hooks at 1733408017365Disabling compacts and flushes for region at 1733408017365Disabling writes for close at 1733408017365Writing region close event to WAL at 1733408017366 (+1 ms)Running coprocessor post-close hooks at 1733408017380 (+14 ms)Closed at 1733408017380 2024-12-05T14:13:37,380 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733408011017.9dd338d2cdec8bcc235bdb212f336d28. 2024-12-05T14:13:37,381 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 710e86e039ffc8d96f214260888d4849, disabling compactions & flushes 2024-12-05T14:13:37,381 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:37,381 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:37,381 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. after waiting 0 ms 2024-12-05T14:13:37,381 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:37,381 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,381 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,382 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741869_1045 (size=8192) 2024-12-05T14:13:37,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741869_1045 (size=8192) 2024-12-05T14:13:37,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741869_1045 (size=8192) 2024-12-05T14:13:37,384 INFO [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/info/b9130c4d5cdd4f639c4a93a5413bc4ae 2024-12-05T14:13:37,386 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,388 INFO [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 88e959e3bb59b49f279be7426309e052; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68964737, jitterRate=0.02765466272830963}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T14:13:37,388 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88e959e3bb59b49f279be7426309e052 2024-12-05T14:13:37,388 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 88e959e3bb59b49f279be7426309e052: Running coprocessor pre-open hook at 1733408017372Writing region info on filesystem at 1733408017372Initializing all the Stores at 1733408017373 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733408017374 (+1 ms)Cleaning up temporary data from old regions at 1733408017382 (+8 ms)Running coprocessor post-open hooks at 1733408017388 (+6 ms)Region opened successfully at 1733408017388 2024-12-05T14:13:37,389 WARN [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 218194e0d8ec,36017,1733408008679 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T14:13:37,389 DEBUG [RS_OPEN_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,qqq,1733408011017.88e959e3bb59b49f279be7426309e052. and can not recover 2024-12-05T14:13:37,392 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/710e86e039ffc8d96f214260888d4849/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,392 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/44e3fd15a9de4a8ceefe8e8505a163f2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,393 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 44e3fd15a9de4a8ceefe8e8505a163f2: Waiting for close lock at 1733408017379Running coprocessor pre-close hooks at 1733408017379Disabling compacts and flushes for region at 1733408017379Disabling writes for close at 1733408017379Writing region close event to WAL at 1733408017387 (+8 ms)Running coprocessor post-close hooks at 1733408017393 (+6 ms)Closed at 1733408017393 2024-12-05T14:13:37,393 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 710e86e039ffc8d96f214260888d4849: Waiting for close lock at 1733408017381Running coprocessor pre-close hooks at 1733408017381Disabling compacts and flushes for region at 1733408017381Disabling writes for close at 1733408017381Writing region close event to WAL at 1733408017385 (+4 ms)Running coprocessor post-close hooks at 1733408017393 (+8 ms)Closed at 1733408017393 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733408011017.44e3fd15a9de4a8ceefe8e8505a163f2. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733408011017.710e86e039ffc8d96f214260888d4849. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 79e2982faca0d2750624de91bb80c08e, disabling compactions & flushes 2024-12-05T14:13:37,393 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 811fb81e434a8467b09b0f283ae0207b, disabling compactions & flushes 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:37,393 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. after waiting 0 ms 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. after waiting 0 ms 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:37,393 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/.tmp/info/b9130c4d5cdd4f639c4a93a5413bc4ae as hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/info/b9130c4d5cdd4f639c4a93a5413bc4ae 2024-12-05T14:13:37,399 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/79e2982faca0d2750624de91bb80c08e/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-12-05T14:13:37,399 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/811fb81e434a8467b09b0f283ae0207b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,399 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 79e2982faca0d2750624de91bb80c08e: Waiting for close lock at 1733408017393Running coprocessor pre-close hooks at 1733408017393Disabling compacts and flushes for region at 1733408017393Disabling writes for close at 1733408017393Writing region close event to WAL at 1733408017393Running coprocessor post-close hooks at 1733408017399 (+6 ms)Closed at 1733408017399 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733408011017.79e2982faca0d2750624de91bb80c08e. 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aba2432d09baa11b4c1ea7026329d127, disabling compactions & flushes 2024-12-05T14:13:37,400 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. after waiting 0 ms 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:37,400 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 811fb81e434a8467b09b0f283ae0207b: Waiting for close lock at 1733408017393Running coprocessor pre-close hooks at 1733408017393Disabling compacts and flushes for region at 1733408017393Disabling writes for close at 1733408017393Writing region close event to WAL at 1733408017394 (+1 ms)Running coprocessor post-close hooks at 1733408017400 (+6 ms)Closed at 1733408017400 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733408011017.811fb81e434a8467b09b0f283ae0207b. 2024-12-05T14:13:37,400 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ffa1b4ffdfeddf082998369feea2e28b, disabling compactions & flushes 2024-12-05T14:13:37,400 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:37,401 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:37,401 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. after waiting 0 ms 2024-12-05T14:13:37,401 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:37,405 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/ffa1b4ffdfeddf082998369feea2e28b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,406 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:37,406 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ffa1b4ffdfeddf082998369feea2e28b: Waiting for close lock at 1733408017400Running coprocessor pre-close hooks at 1733408017400Disabling compacts and flushes for region at 1733408017400Disabling writes for close at 1733408017401 (+1 ms)Writing region close event to WAL at 1733408017401Running coprocessor post-close hooks at 1733408017406 (+5 ms)Closed at 1733408017406 2024-12-05T14:13:37,406 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733408011017.ffa1b4ffdfeddf082998369feea2e28b. 2024-12-05T14:13:37,406 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 70c6030455907dcfb74337bc850642ff, disabling compactions & flushes 2024-12-05T14:13:37,406 INFO [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/info/b9130c4d5cdd4f639c4a93a5413bc4ae, entries=24, sequenceid=75, filesize=8 K 2024-12-05T14:13:37,406 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:37,407 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:37,407 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. after waiting 0 ms 2024-12-05T14:13:37,407 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:37,407 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/aba2432d09baa11b4c1ea7026329d127/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,408 INFO [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 58ms, sequenceid=75, compaction requested=false 2024-12-05T14:13:37,408 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:37,408 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aba2432d09baa11b4c1ea7026329d127: Waiting for close lock at 1733408017400Running coprocessor pre-close hooks at 1733408017400Disabling compacts and flushes for region at 1733408017400Disabling writes for close at 1733408017400Writing region close event to WAL at 1733408017401 (+1 ms)Running coprocessor post-close hooks at 1733408017408 (+7 ms)Closed at 1733408017408 2024-12-05T14:13:37,409 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733408011017.aba2432d09baa11b4c1ea7026329d127. 2024-12-05T14:13:37,409 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4edea3efe3103a410142665ac1ea3d83, disabling compactions & flushes 2024-12-05T14:13:37,409 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:37,409 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:37,409 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. after waiting 0 ms 2024-12-05T14:13:37,409 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:37,414 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/70c6030455907dcfb74337bc850642ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,415 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:37,415 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 70c6030455907dcfb74337bc850642ff: Waiting for close lock at 1733408017406Running coprocessor pre-close hooks at 1733408017406Disabling compacts and flushes for region at 1733408017406Disabling writes for close at 1733408017407 (+1 ms)Writing region close event to WAL at 1733408017407Running coprocessor post-close hooks at 1733408017415 (+8 ms)Closed at 1733408017415 2024-12-05T14:13:37,415 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733408011017.70c6030455907dcfb74337bc850642ff. 2024-12-05T14:13:37,415 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/4edea3efe3103a410142665ac1ea3d83/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,415 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88bd874c8e3a659802f7bbb662036a12, disabling compactions & flushes 2024-12-05T14:13:37,415 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:37,415 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:37,415 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. after waiting 0 ms 2024-12-05T14:13:37,415 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:37,416 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:37,416 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4edea3efe3103a410142665ac1ea3d83: Waiting for close lock at 1733408017409Running coprocessor pre-close hooks at 1733408017409Disabling compacts and flushes for region at 1733408017409Disabling writes for close at 1733408017409Writing region close event to WAL at 1733408017409Running coprocessor post-close hooks at 1733408017416 (+7 ms)Closed at 1733408017416 2024-12-05T14:13:37,416 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733408011017.4edea3efe3103a410142665ac1ea3d83. 2024-12-05T14:13:37,416 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd727192dbe84371a93aad39b906dd3b, disabling compactions & flushes 2024-12-05T14:13:37,416 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:37,416 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:37,416 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. after waiting 0 ms 2024-12-05T14:13:37,416 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:37,417 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-12-05T14:13:37,419 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T14:13:37,419 INFO [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T14:13:37,419 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733408017350Running coprocessor pre-close hooks at 1733408017350Disabling compacts and flushes for region at 1733408017350Disabling writes for close at 1733408017350Obtaining lock to block concurrent updates at 1733408017350Preparing flush snapshotting stores in 1588230740 at 1733408017350Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1733408017357 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733408017358 (+1 ms)Flushing 1588230740/info: creating writer at 1733408017358Flushing 1588230740/info: appending metadata at 1733408017369 (+11 ms)Flushing 1588230740/info: closing flushed file at 1733408017369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@616f9c91: reopening flushed file at 1733408017392 (+23 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 58ms, sequenceid=75, compaction requested=false at 1733408017408 (+16 ms)Writing region close event to WAL at 1733408017411 (+3 ms)Running coprocessor post-close hooks at 1733408017418 (+7 ms)Closed at 1733408017419 (+1 ms) 2024-12-05T14:13:37,419 DEBUG [RS_CLOSE_META-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T14:13:37,420 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/88bd874c8e3a659802f7bbb662036a12/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,421 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:37,421 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/cd727192dbe84371a93aad39b906dd3b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,421 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88bd874c8e3a659802f7bbb662036a12: Waiting for close lock at 1733408017415Running coprocessor pre-close hooks at 1733408017415Disabling compacts and flushes for region at 1733408017415Disabling writes for close at 1733408017415Writing region close event to WAL at 1733408017416 (+1 ms)Running coprocessor post-close hooks at 1733408017421 (+5 ms)Closed at 1733408017421 2024-12-05T14:13:37,421 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1733408011017.88bd874c8e3a659802f7bbb662036a12. 2024-12-05T14:13:37,421 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c14601cce05aed77304a5058eff93ba1, disabling compactions & flushes 2024-12-05T14:13:37,421 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:37,421 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:37,421 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. after waiting 0 ms 2024-12-05T14:13:37,421 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:37,421 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:37,422 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd727192dbe84371a93aad39b906dd3b: Waiting for close lock at 1733408017416Running coprocessor pre-close hooks at 1733408017416Disabling compacts and flushes for region at 1733408017416Disabling writes for close at 1733408017416Writing region close event to WAL at 1733408017417 (+1 ms)Running coprocessor post-close hooks at 1733408017421 (+4 ms)Closed at 1733408017421 2024-12-05T14:13:37,422 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733408011017.cd727192dbe84371a93aad39b906dd3b. 2024-12-05T14:13:37,422 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 564487ebab3bd815a85a50262b275a91, disabling compactions & flushes 2024-12-05T14:13:37,422 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:37,422 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:37,422 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. after waiting 0 ms 2024-12-05T14:13:37,422 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:37,425 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/564487ebab3bd815a85a50262b275a91/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,425 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c14601cce05aed77304a5058eff93ba1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,426 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c14601cce05aed77304a5058eff93ba1: Waiting for close lock at 1733408017421Running coprocessor pre-close hooks at 1733408017421Disabling compacts and flushes for region at 1733408017421Disabling writes for close at 1733408017421Writing region close event to WAL at 1733408017422 (+1 ms)Running coprocessor post-close hooks at 1733408017426 (+4 ms)Closed at 1733408017426 2024-12-05T14:13:37,426 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 564487ebab3bd815a85a50262b275a91: Waiting for close lock at 1733408017422Running coprocessor pre-close hooks at 1733408017422Disabling compacts and flushes for region at 1733408017422Disabling writes for close at 1733408017422Writing region close event to WAL at 1733408017422Running coprocessor post-close hooks at 1733408017426 (+4 ms)Closed at 1733408017426 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733408011017.c14601cce05aed77304a5058eff93ba1. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733408011017.564487ebab3bd815a85a50262b275a91. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a24b719bda0e807d3ebf3858def7531b, disabling compactions & flushes 2024-12-05T14:13:37,426 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5c703cd272011369700660d18171711b, disabling compactions & flushes 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. after waiting 0 ms 2024-12-05T14:13:37,426 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733408011017.5c703cd272011369700660d18171711b. after waiting 0 ms 2024-12-05T14:13:37,426 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:37,430 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/5c703cd272011369700660d18171711b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,430 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/a24b719bda0e807d3ebf3858def7531b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,431 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:37,431 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5c703cd272011369700660d18171711b: Waiting for close lock at 1733408017426Running coprocessor pre-close hooks at 1733408017426Disabling compacts and flushes for region at 1733408017426Disabling writes for close at 1733408017426Writing region close event to WAL at 1733408017426Running coprocessor post-close hooks at 1733408017431 (+5 ms)Closed at 1733408017431 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a24b719bda0e807d3ebf3858def7531b: Waiting for close lock at 1733408017426Running coprocessor pre-close hooks at 1733408017426Disabling compacts and flushes for region at 1733408017426Disabling writes for close at 1733408017426Writing region close event to WAL at 1733408017426Running coprocessor post-close hooks at 1733408017431 (+5 ms)Closed at 1733408017431 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733408011017.5c703cd272011369700660d18171711b. 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733408011017.a24b719bda0e807d3ebf3858def7531b. 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b620ca010e58c2eb2f2309f4a11c2dba, disabling compactions & flushes 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 59e732700952e1f67c04f79b391abf8c, disabling compactions & flushes 2024-12-05T14:13:37,431 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:37,431 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. after waiting 0 ms 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. after waiting 0 ms 2024-12-05T14:13:37,431 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:37,436 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/59e732700952e1f67c04f79b391abf8c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,436 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/b620ca010e58c2eb2f2309f4a11c2dba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,436 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:37,436 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:37,436 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 59e732700952e1f67c04f79b391abf8c: Waiting for close lock at 1733408017431Running coprocessor pre-close hooks at 1733408017431Disabling compacts and flushes for region at 1733408017431Disabling writes for close at 1733408017431Writing region close event to WAL at 1733408017432 (+1 ms)Running coprocessor post-close hooks at 1733408017436 (+4 ms)Closed at 1733408017436 2024-12-05T14:13:37,436 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b620ca010e58c2eb2f2309f4a11c2dba: Waiting for close lock at 1733408017431Running coprocessor pre-close hooks at 1733408017431Disabling compacts and flushes for region at 1733408017431Disabling writes for close at 1733408017431Writing region close event to WAL at 1733408017432 (+1 ms)Running coprocessor post-close hooks at 1733408017436 (+4 ms)Closed at 1733408017436 2024-12-05T14:13:37,437 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733408011017.59e732700952e1f67c04f79b391abf8c. 2024-12-05T14:13:37,437 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733408011017.b620ca010e58c2eb2f2309f4a11c2dba. 2024-12-05T14:13:37,437 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c129ce5c0755b4946bdefc2d36f50149, disabling compactions & flushes 2024-12-05T14:13:37,437 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:37,437 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:37,437 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. after waiting 0 ms 2024-12-05T14:13:37,437 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:37,443 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/c129ce5c0755b4946bdefc2d36f50149/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,444 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:37,444 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c129ce5c0755b4946bdefc2d36f50149: Waiting for close lock at 1733408017437Running coprocessor pre-close hooks at 1733408017437Disabling compacts and flushes for region at 1733408017437Disabling writes for close at 1733408017437Writing region close event to WAL at 1733408017437Running coprocessor post-close hooks at 1733408017444 (+7 ms)Closed at 1733408017444 2024-12-05T14:13:37,444 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733408011017.c129ce5c0755b4946bdefc2d36f50149. 2024-12-05T14:13:37,444 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 894d0c86da04798f8fea0640c4aa6d05, disabling compactions & flushes 2024-12-05T14:13:37,444 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:37,444 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:37,444 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. after waiting 0 ms 2024-12-05T14:13:37,444 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:37,449 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/data/default/test/894d0c86da04798f8fea0640c4aa6d05/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T14:13:37,449 INFO [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:37,449 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 894d0c86da04798f8fea0640c4aa6d05: Waiting for close lock at 1733408017444Running coprocessor pre-close hooks at 1733408017444Disabling compacts and flushes for region at 1733408017444Disabling writes for close at 1733408017444Writing region close event to WAL at 1733408017445 (+1 ms)Running coprocessor post-close hooks at 1733408017449 (+4 ms)Closed at 1733408017449 2024-12-05T14:13:37,449 DEBUG [RS_CLOSE_REGION-regionserver/218194e0d8ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733408011017.894d0c86da04798f8fea0640c4aa6d05. 2024-12-05T14:13:37,550 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.HRegionServer(976): stopping server 218194e0d8ec,45449,1733408008631; all regions closed. 2024-12-05T14:13:37,550 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.HRegionServer(976): stopping server 218194e0d8ec,36017,1733408008679; all regions closed. 2024-12-05T14:13:37,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741865_1041 (size=4201) 2024-12-05T14:13:37,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741834_1010 (size=7074) 2024-12-05T14:13:37,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741834_1010 (size=7074) 2024-12-05T14:13:37,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741865_1041 (size=4201) 2024-12-05T14:13:37,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741865_1041 (size=4201) 2024-12-05T14:13:37,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741834_1010 (size=7074) 2024-12-05T14:13:37,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741833_1009 (size=6526) 2024-12-05T14:13:37,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741833_1009 (size=6526) 2024-12-05T14:13:37,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741833_1009 (size=6526) 2024-12-05T14:13:37,563 DEBUG [RS:2;218194e0d8ec:36017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:37,563 INFO [RS:2;218194e0d8ec:36017 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T14:13:37,563 DEBUG [RS:1;218194e0d8ec:45449 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/oldWALs 2024-12-05T14:13:37,563 INFO [RS:2;218194e0d8ec:36017 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T14:13:37,563 INFO [RS:1;218194e0d8ec:45449 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 218194e0d8ec%2C45449%2C1733408008631:(num 1733408010132) 2024-12-05T14:13:37,563 DEBUG [RS:1;218194e0d8ec:45449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T14:13:37,563 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T14:13:37,563 INFO [RS:1;218194e0d8ec:45449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T14:13:37,563 INFO [RS:2;218194e0d8ec:36017 {}] hbase.ChoreService(370): Chore service for: regionserver/218194e0d8ec:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T14:13:37,563 INFO [RS:1;218194e0d8ec:45449 {}] hbase.ChoreService(370): Chore service for: regionserver/218194e0d8ec:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T14:13:37,563 INFO [RS:2;218194e0d8ec:36017 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T14:13:37,563 INFO [regionserver/218194e0d8ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T14:13:37,563 INFO [RS:2;218194e0d8ec:36017 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36017 2024-12-05T14:13:37,563 INFO [regionserver/218194e0d8ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T14:13:37,563 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T14:13:37,564 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T14:13:37,564 INFO [RS:1;218194e0d8ec:45449 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T14:13:37,564 INFO [RS:1;218194e0d8ec:45449 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T14:13:37,564 INFO [RS:1;218194e0d8ec:45449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45449 2024-12-05T14:13:37,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36017-0x101a5d71eb70003, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/218194e0d8ec,36017,1733408008679 2024-12-05T14:13:37,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T14:13:37,572 INFO [RS:2;218194e0d8ec:36017 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T14:13:37,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45449-0x101a5d71eb70002, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/218194e0d8ec,45449,1733408008631 2024-12-05T14:13:37,572 INFO [RS:1;218194e0d8ec:45449 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T14:13:37,572 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [218194e0d8ec,45449,1733408008631] 2024-12-05T14:13:37,593 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/218194e0d8ec,45449,1733408008631 already deleted, retry=false 2024-12-05T14:13:37,593 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 218194e0d8ec,45449,1733408008631 expired; onlineServers=1 2024-12-05T14:13:37,593 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [218194e0d8ec,36017,1733408008679] 2024-12-05T14:13:37,603 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/218194e0d8ec,36017,1733408008679 already deleted, retry=false 2024-12-05T14:13:37,603 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 218194e0d8ec,36017,1733408008679 expired; onlineServers=0 2024-12-05T14:13:37,603 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '218194e0d8ec,38561,1733408007726' ***** 2024-12-05T14:13:37,603 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T14:13:37,603 INFO [M:0;218194e0d8ec:38561 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T14:13:37,604 INFO [M:0;218194e0d8ec:38561 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T14:13:37,604 DEBUG [M:0;218194e0d8ec:38561 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T14:13:37,604 DEBUG [M:0;218194e0d8ec:38561 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T14:13:37,604 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T14:13:37,604 DEBUG [master/218194e0d8ec:0:becomeActiveMaster-HFileCleaner.large.0-1733408009806 {}] cleaner.HFileCleaner(306): Exit Thread[master/218194e0d8ec:0:becomeActiveMaster-HFileCleaner.large.0-1733408009806,5,FailOnTimeoutGroup] 2024-12-05T14:13:37,604 DEBUG [master/218194e0d8ec:0:becomeActiveMaster-HFileCleaner.small.0-1733408009807 {}] cleaner.HFileCleaner(306): Exit Thread[master/218194e0d8ec:0:becomeActiveMaster-HFileCleaner.small.0-1733408009807,5,FailOnTimeoutGroup] 2024-12-05T14:13:37,604 INFO [M:0;218194e0d8ec:38561 {}] hbase.ChoreService(370): Chore service for: master/218194e0d8ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T14:13:37,604 INFO [M:0;218194e0d8ec:38561 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T14:13:37,605 DEBUG [M:0;218194e0d8ec:38561 {}] master.HMaster(1795): Stopping service threads 2024-12-05T14:13:37,605 INFO [M:0;218194e0d8ec:38561 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T14:13:37,605 INFO [M:0;218194e0d8ec:38561 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T14:13:37,605 INFO [M:0;218194e0d8ec:38561 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T14:13:37,605 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T14:13:37,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T14:13:37,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T14:13:37,614 DEBUG [M:0;218194e0d8ec:38561 {}] zookeeper.ZKUtil(347): master:38561-0x101a5d71eb70000, quorum=127.0.0.1:62933, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T14:13:37,614 WARN [M:0;218194e0d8ec:38561 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T14:13:37,615 INFO [M:0;218194e0d8ec:38561 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/.lastflushedseqids 2024-12-05T14:13:37,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741870_1046 (size=1) 2024-12-05T14:13:37,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741870_1046 (size=1) 2024-12-05T14:13:37,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741870_1046 (size=1) 2024-12-05T14:13:37,630 INFO [M:0;218194e0d8ec:38561 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T14:13:37,630 INFO [M:0;218194e0d8ec:38561 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T14:13:37,630 DEBUG [M:0;218194e0d8ec:38561 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T14:13:37,630 INFO [M:0;218194e0d8ec:38561 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:13:37,630 DEBUG [M:0;218194e0d8ec:38561 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:13:37,630 DEBUG [M:0;218194e0d8ec:38561 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T14:13:37,630 DEBUG [M:0;218194e0d8ec:38561 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T14:13:37,630 INFO [M:0;218194e0d8ec:38561 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.49 KB heapSize=315.40 KB 2024-12-05T14:13:37,646 DEBUG [M:0;218194e0d8ec:38561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/561ab4c4839b42488e34f793f48a9cc8 is 82, key is hbase:meta,,1/info:regioninfo/1733408015585/Put/seqid=0 2024-12-05T14:13:37,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44997 is added to blk_1073741871_1047 (size=6071) 2024-12-05T14:13:37,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32959 is added to blk_1073741871_1047 (size=6071) 2024-12-05T14:13:37,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38807 is added to blk_1073741871_1047 (size=6071) 2024-12-05T14:13:37,653 INFO [M:0;218194e0d8ec:38561 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/561ab4c4839b42488e34f793f48a9cc8 2024-12-05T14:13:37,675 DEBUG [M:0;218194e0d8ec:38561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42699/user/jenkins/test-data/6260465d-ce73-df9b-4c90-30936b9c6b41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0795ae9e86c24ab4abd51816b47f521b is 2987, key is \x00\x00\x00\x00\x00\x00\x00