2024-11-20 15:04:59,394 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-20 15:04:59,407 main DEBUG Took 0.010480 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 15:04:59,407 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 15:04:59,407 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 15:04:59,408 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 15:04:59,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,417 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 15:04:59,431 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,434 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,435 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,437 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,438 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,439 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,440 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,441 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,442 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,443 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,444 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,445 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 15:04:59,446 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,447 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 15:04:59,448 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 15:04:59,450 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 15:04:59,452 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 15:04:59,453 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 15:04:59,454 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 15:04:59,455 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 15:04:59,463 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 15:04:59,465 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 15:04:59,466 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 15:04:59,467 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 15:04:59,467 main DEBUG createAppenders(={Console}) 2024-11-20 15:04:59,468 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-20 15:04:59,468 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-20 15:04:59,468 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-20 15:04:59,469 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 15:04:59,469 main DEBUG OutputStream closed 2024-11-20 15:04:59,469 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 15:04:59,470 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 15:04:59,470 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-20 15:04:59,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 15:04:59,539 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 15:04:59,540 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 15:04:59,541 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 15:04:59,541 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 15:04:59,541 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 15:04:59,542 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 15:04:59,542 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 15:04:59,542 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 15:04:59,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 15:04:59,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 15:04:59,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 15:04:59,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 15:04:59,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 15:04:59,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 15:04:59,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 15:04:59,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 15:04:59,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 15:04:59,547 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 15:04:59,548 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-20 15:04:59,548 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 15:04:59,549 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-20T15:04:59,561 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-20 15:04:59,563 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 15:04:59,564 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T15:04:59,767 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18 2024-11-20T15:04:59,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T15:04:59,799 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a, deleteOnExit=true 2024-11-20T15:04:59,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T15:04:59,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/test.cache.data in system properties and HBase conf 2024-11-20T15:04:59,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T15:04:59,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/hadoop.log.dir in system properties and HBase conf 2024-11-20T15:04:59,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T15:04:59,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T15:04:59,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T15:04:59,888 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T15:04:59,981 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T15:04:59,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T15:04:59,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T15:04:59,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T15:04:59,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T15:04:59,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T15:04:59,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T15:04:59,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T15:04:59,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T15:04:59,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T15:04:59,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/nfs.dump.dir in system properties and HBase conf 2024-11-20T15:04:59,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/java.io.tmpdir in system properties and HBase conf 2024-11-20T15:04:59,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T15:04:59,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T15:04:59,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T15:05:01,051 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T15:05:01,120 INFO [Time-limited test {}] log.Log(170): Logging initialized @2356ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T15:05:01,185 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T15:05:01,242 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T15:05:01,266 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T15:05:01,266 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T15:05:01,268 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T15:05:01,282 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T15:05:01,285 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/hadoop.log.dir/,AVAILABLE} 2024-11-20T15:05:01,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T15:05:01,481 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/java.io.tmpdir/jetty-localhost-43681-hadoop-hdfs-3_4_1-tests_jar-_-any-7259683560460413506/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T15:05:01,491 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:43681} 2024-11-20T15:05:01,491 INFO [Time-limited test {}] server.Server(415): Started @2729ms 2024-11-20T15:05:01,999 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T15:05:02,008 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T15:05:02,011 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T15:05:02,011 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T15:05:02,011 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T15:05:02,012 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/hadoop.log.dir/,AVAILABLE} 2024-11-20T15:05:02,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T15:05:02,112 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/java.io.tmpdir/jetty-localhost-40001-hadoop-hdfs-3_4_1-tests_jar-_-any-16269478119614072403/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T15:05:02,113 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:40001} 2024-11-20T15:05:02,113 INFO [Time-limited test {}] server.Server(415): Started @3351ms 2024-11-20T15:05:02,161 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T15:05:02,273 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T15:05:02,280 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T15:05:02,281 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T15:05:02,281 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T15:05:02,282 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T15:05:02,283 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/hadoop.log.dir/,AVAILABLE} 2024-11-20T15:05:02,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T15:05:02,387 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/java.io.tmpdir/jetty-localhost-36535-hadoop-hdfs-3_4_1-tests_jar-_-any-16522447504563139728/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T15:05:02,387 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:36535} 2024-11-20T15:05:02,387 INFO [Time-limited test {}] server.Server(415): Started @3625ms 2024-11-20T15:05:02,390 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T15:05:02,431 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T15:05:02,436 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T15:05:02,437 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T15:05:02,438 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T15:05:02,438 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T15:05:02,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/hadoop.log.dir/,AVAILABLE} 2024-11-20T15:05:02,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T15:05:02,537 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/java.io.tmpdir/jetty-localhost-46631-hadoop-hdfs-3_4_1-tests_jar-_-any-2872915325909600665/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T15:05:02,538 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:46631} 2024-11-20T15:05:02,539 INFO [Time-limited test {}] server.Server(415): Started @3776ms 2024-11-20T15:05:02,541 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T15:05:03,526 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/data/data3/current/BP-1123765204-172.17.0.2-1732115100595/current, will proceed with Du for space computation calculation, 2024-11-20T15:05:03,526 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/data/data1/current/BP-1123765204-172.17.0.2-1732115100595/current, will proceed with Du for space computation calculation, 2024-11-20T15:05:03,526 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/data/data4/current/BP-1123765204-172.17.0.2-1732115100595/current, will proceed with Du for space computation calculation, 2024-11-20T15:05:03,526 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/data/data2/current/BP-1123765204-172.17.0.2-1732115100595/current, will proceed with Du for space computation calculation, 2024-11-20T15:05:03,562 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T15:05:03,562 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T15:05:03,590 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/data/data5/current/BP-1123765204-172.17.0.2-1732115100595/current, will proceed with Du for space computation calculation, 2024-11-20T15:05:03,590 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/data/data6/current/BP-1123765204-172.17.0.2-1732115100595/current, will proceed with Du for space computation calculation, 2024-11-20T15:05:03,609 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe10d280c73241eb0 with lease ID 0xf70eb00cf52d8e23: Processing first storage report for DS-a0b7ff8b-cc8f-4112-bea6-13ca8f08a4a2 from datanode DatanodeRegistration(127.0.0.1:38935, datanodeUuid=e0de6775-232a-4276-b74e-06b362f0632e, infoPort=42775, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595) 2024-11-20T15:05:03,610 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe10d280c73241eb0 with lease ID 0xf70eb00cf52d8e23: from storage DS-a0b7ff8b-cc8f-4112-bea6-13ca8f08a4a2 node DatanodeRegistration(127.0.0.1:38935, datanodeUuid=e0de6775-232a-4276-b74e-06b362f0632e, infoPort=42775, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T15:05:03,610 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x544d38debe2d5dad with lease ID 0xf70eb00cf52d8e24: Processing first storage report for DS-6246c99a-c186-42f4-960c-91269a275f0d from datanode DatanodeRegistration(127.0.0.1:34807, datanodeUuid=8090d9ef-8b34-4119-9a27-8d6cbd0a1f44, infoPort=44829, infoSecurePort=0, ipcPort=39385, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595) 2024-11-20T15:05:03,610 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x544d38debe2d5dad with lease ID 0xf70eb00cf52d8e24: from storage DS-6246c99a-c186-42f4-960c-91269a275f0d node DatanodeRegistration(127.0.0.1:34807, datanodeUuid=8090d9ef-8b34-4119-9a27-8d6cbd0a1f44, infoPort=44829, infoSecurePort=0, ipcPort=39385, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T15:05:03,610 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe10d280c73241eb0 with lease ID 0xf70eb00cf52d8e23: Processing first storage report for DS-a6676ca9-dbea-4dd6-94e7-5501674d3ce1 from datanode DatanodeRegistration(127.0.0.1:38935, datanodeUuid=e0de6775-232a-4276-b74e-06b362f0632e, infoPort=42775, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595) 2024-11-20T15:05:03,611 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe10d280c73241eb0 with lease ID 0xf70eb00cf52d8e23: from storage DS-a6676ca9-dbea-4dd6-94e7-5501674d3ce1 node DatanodeRegistration(127.0.0.1:38935, datanodeUuid=e0de6775-232a-4276-b74e-06b362f0632e, infoPort=42775, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T15:05:03,611 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x544d38debe2d5dad with lease ID 0xf70eb00cf52d8e24: Processing first storage report for DS-20a79af1-5938-4a60-9cea-e8d32353d56d from datanode DatanodeRegistration(127.0.0.1:34807, datanodeUuid=8090d9ef-8b34-4119-9a27-8d6cbd0a1f44, infoPort=44829, infoSecurePort=0, ipcPort=39385, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595) 2024-11-20T15:05:03,611 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x544d38debe2d5dad with lease ID 0xf70eb00cf52d8e24: from storage DS-20a79af1-5938-4a60-9cea-e8d32353d56d node DatanodeRegistration(127.0.0.1:34807, datanodeUuid=8090d9ef-8b34-4119-9a27-8d6cbd0a1f44, infoPort=44829, infoSecurePort=0, ipcPort=39385, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T15:05:03,614 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T15:05:03,619 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x52a8a3b2f27c1475 with lease ID 0xf70eb00cf52d8e25: Processing first storage report for DS-77d67676-1e5f-4a3d-9c70-d4f0caedd6ad from datanode DatanodeRegistration(127.0.0.1:42487, datanodeUuid=538b26f2-a9d8-46ce-8118-7df405d3bfc2, infoPort=42085, infoSecurePort=0, ipcPort=40081, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595) 2024-11-20T15:05:03,619 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x52a8a3b2f27c1475 with lease ID 0xf70eb00cf52d8e25: from storage DS-77d67676-1e5f-4a3d-9c70-d4f0caedd6ad node DatanodeRegistration(127.0.0.1:42487, datanodeUuid=538b26f2-a9d8-46ce-8118-7df405d3bfc2, infoPort=42085, infoSecurePort=0, ipcPort=40081, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T15:05:03,619 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x52a8a3b2f27c1475 with lease ID 0xf70eb00cf52d8e25: Processing first storage report for DS-29035557-c0e6-48e6-bdb5-9253e7dd3c4e from datanode DatanodeRegistration(127.0.0.1:42487, datanodeUuid=538b26f2-a9d8-46ce-8118-7df405d3bfc2, infoPort=42085, infoSecurePort=0, ipcPort=40081, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595) 2024-11-20T15:05:03,620 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x52a8a3b2f27c1475 with lease ID 0xf70eb00cf52d8e25: from storage DS-29035557-c0e6-48e6-bdb5-9253e7dd3c4e node DatanodeRegistration(127.0.0.1:42487, datanodeUuid=538b26f2-a9d8-46ce-8118-7df405d3bfc2, infoPort=42085, infoSecurePort=0, ipcPort=40081, storageInfo=lv=-57;cid=testClusterID;nsid=426318239;c=1732115100595), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T15:05:03,643 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18 2024-11-20T15:05:03,704 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/zookeeper_0, clientPort=56692, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b314ce72-b7f6-ec9f-c3bb-8355a4aaaa18/cluster_d0f740de-e77d-fa67-578e-b6ef1dd9459a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T15:05:03,714 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56692 2024-11-20T15:05:03,724 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:03,726 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:03,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741825_1001 (size=7) 2024-11-20T15:05:03,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741825_1001 (size=7) 2024-11-20T15:05:03,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741825_1001 (size=7) 2024-11-20T15:05:04,342 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a with version=8 2024-11-20T15:05:04,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/hbase-staging 2024-11-20T15:05:04,423 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T15:05:04,657 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T15:05:04,666 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:04,666 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:04,671 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T15:05:04,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:04,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T15:05:04,794 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T15:05:04,846 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T15:05:04,854 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T15:05:04,857 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T15:05:04,878 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 105691 (auto-detected) 2024-11-20T15:05:04,879 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T15:05:04,895 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39091 2024-11-20T15:05:04,915 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39091 connecting to ZooKeeper ensemble=127.0.0.1:56692 2024-11-20T15:05:05,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390910x0, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T15:05:05,035 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39091-0x1015a10928d0000 connected 2024-11-20T15:05:05,112 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,115 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,124 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T15:05:05,128 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a, hbase.cluster.distributed=false 2024-11-20T15:05:05,150 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T15:05:05,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39091 2024-11-20T15:05:05,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39091 2024-11-20T15:05:05,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39091 2024-11-20T15:05:05,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39091 2024-11-20T15:05:05,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39091 2024-11-20T15:05:05,253 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T15:05:05,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,255 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T15:05:05,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T15:05:05,258 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T15:05:05,261 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T15:05:05,262 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42753 2024-11-20T15:05:05,264 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42753 connecting to ZooKeeper ensemble=127.0.0.1:56692 2024-11-20T15:05:05,265 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,269 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427530x0, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T15:05:05,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427530x0, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T15:05:05,303 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42753-0x1015a10928d0001 connected 2024-11-20T15:05:05,308 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T15:05:05,319 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T15:05:05,321 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T15:05:05,326 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T15:05:05,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42753 2024-11-20T15:05:05,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42753 2024-11-20T15:05:05,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42753 2024-11-20T15:05:05,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42753 2024-11-20T15:05:05,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42753 2024-11-20T15:05:05,345 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T15:05:05,345 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,345 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,346 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T15:05:05,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T15:05:05,346 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T15:05:05,346 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T15:05:05,347 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39199 2024-11-20T15:05:05,349 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39199 connecting to ZooKeeper ensemble=127.0.0.1:56692 2024-11-20T15:05:05,350 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,353 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391990x0, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T15:05:05,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:391990x0, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T15:05:05,369 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39199-0x1015a10928d0002 connected 2024-11-20T15:05:05,370 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T15:05:05,370 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T15:05:05,371 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T15:05:05,373 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T15:05:05,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39199 2024-11-20T15:05:05,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39199 2024-11-20T15:05:05,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39199 2024-11-20T15:05:05,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39199 2024-11-20T15:05:05,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39199 2024-11-20T15:05:05,395 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1a15ecfd95f4:0 server-side Connection retries=45 2024-11-20T15:05:05,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,396 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T15:05:05,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T15:05:05,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T15:05:05,396 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T15:05:05,396 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T15:05:05,397 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36179 2024-11-20T15:05:05,399 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36179 connecting to ZooKeeper ensemble=127.0.0.1:56692 2024-11-20T15:05:05,400 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,402 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361790x0, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T15:05:05,410 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36179-0x1015a10928d0003 connected 2024-11-20T15:05:05,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T15:05:05,411 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T15:05:05,413 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T15:05:05,414 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T15:05:05,416 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T15:05:05,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36179 2024-11-20T15:05:05,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36179 2024-11-20T15:05:05,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36179 2024-11-20T15:05:05,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36179 2024-11-20T15:05:05,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36179 2024-11-20T15:05:05,437 DEBUG [M:0;1a15ecfd95f4:39091 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a15ecfd95f4:39091 2024-11-20T15:05:05,438 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:05,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T15:05:05,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T15:05:05,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T15:05:05,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T15:05:05,455 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:05,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:05,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T15:05:05,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T15:05:05,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T15:05:05,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:05,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:05,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:05,486 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T15:05:05,487 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a15ecfd95f4,39091,1732115104505 from backup master directory 2024-11-20T15:05:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T15:05:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T15:05:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T15:05:05,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T15:05:05,494 WARN [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T15:05:05,494 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:05,497 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T15:05:05,498 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T15:05:05,549 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/hbase.id] with ID: 86152321-5c1e-4b52-a667-0afbaf8c63e9 2024-11-20T15:05:05,549 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/.tmp/hbase.id 2024-11-20T15:05:05,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741826_1002 (size=42) 2024-11-20T15:05:05,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741826_1002 (size=42) 2024-11-20T15:05:05,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741826_1002 (size=42) 2024-11-20T15:05:05,566 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/.tmp/hbase.id]:[hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/hbase.id] 2024-11-20T15:05:05,614 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T15:05:05,620 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T15:05:05,638 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-20T15:05:05,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:05,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:05,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:05,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:05,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741827_1003 (size=196) 2024-11-20T15:05:05,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741827_1003 (size=196) 2024-11-20T15:05:05,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741827_1003 (size=196) 2024-11-20T15:05:05,703 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T15:05:05,705 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T15:05:05,717 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T15:05:05,720 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T15:05:05,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741828_1004 (size=1189) 2024-11-20T15:05:05,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741828_1004 (size=1189) 2024-11-20T15:05:05,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741828_1004 (size=1189) 2024-11-20T15:05:05,768 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/data/master/store 2024-11-20T15:05:05,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741829_1005 (size=34) 2024-11-20T15:05:05,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741829_1005 (size=34) 2024-11-20T15:05:05,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741829_1005 (size=34) 2024-11-20T15:05:05,793 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T15:05:05,796 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:05,797 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T15:05:05,798 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T15:05:05,798 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T15:05:05,799 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T15:05:05,799 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T15:05:05,799 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T15:05:05,801 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732115105797Disabling compacts and flushes for region at 1732115105797Disabling writes for close at 1732115105799 (+2 ms)Writing region close event to WAL at 1732115105799Closed at 1732115105799 2024-11-20T15:05:05,803 WARN [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/data/master/store/.initializing 2024-11-20T15:05:05,803 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/WALs/1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:05,811 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T15:05:05,824 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a15ecfd95f4%2C39091%2C1732115104505, suffix=, logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/WALs/1a15ecfd95f4,39091,1732115104505, archiveDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/oldWALs, maxLogs=10 2024-11-20T15:05:05,845 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/WALs/1a15ecfd95f4,39091,1732115104505/1a15ecfd95f4%2C39091%2C1732115104505.1732115105828, exclude list is [], retry=0 2024-11-20T15:05:05,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42487,DS-77d67676-1e5f-4a3d-9c70-d4f0caedd6ad,DISK] 2024-11-20T15:05:05,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34807,DS-6246c99a-c186-42f4-960c-91269a275f0d,DISK] 2024-11-20T15:05:05,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38935,DS-a0b7ff8b-cc8f-4112-bea6-13ca8f08a4a2,DISK] 2024-11-20T15:05:05,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T15:05:05,900 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/WALs/1a15ecfd95f4,39091,1732115104505/1a15ecfd95f4%2C39091%2C1732115104505.1732115105828 2024-11-20T15:05:05,900 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42085:42085),(127.0.0.1/127.0.0.1:44829:44829),(127.0.0.1/127.0.0.1:42775:42775)] 2024-11-20T15:05:05,901 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T15:05:05,901 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:05,904 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,905 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T15:05:05,964 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:05,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:05,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,971 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T15:05:05,971 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:05,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:05,973 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T15:05:05,976 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:05,977 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:05,977 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,980 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T15:05:05,980 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:05,982 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:05,982 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,985 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,986 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,992 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,992 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:05,996 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T15:05:06,000 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T15:05:06,004 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:06,005 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72315108, jitterRate=0.0775790810585022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T15:05:06,012 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732115105916Initializing all the Stores at 1732115105918 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115105918Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115105919 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115105919Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115105919Cleaning up temporary data from old regions at 1732115105992 (+73 ms)Region opened successfully at 1732115106011 (+19 ms) 2024-11-20T15:05:06,013 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T15:05:06,043 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3383b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T15:05:06,069 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T15:05:06,078 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:06,078 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:06,080 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:06,082 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T15:05:06,086 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T15:05:06,086 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T15:05:06,108 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T15:05:06,116 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T15:05:06,151 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T15:05:06,154 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T15:05:06,156 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T15:05:06,168 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T15:05:06,170 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T15:05:06,173 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T15:05:06,184 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T15:05:06,186 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T15:05:06,193 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T15:05:06,213 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T15:05:06,217 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T15:05:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T15:05:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T15:05:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T15:05:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T15:05:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,230 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1a15ecfd95f4,39091,1732115104505, sessionid=0x1015a10928d0000, setting cluster-up flag (Was=false) 2024-11-20T15:05:06,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,285 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T15:05:06,290 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:06,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:06,334 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T15:05:06,338 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:06,348 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T15:05:06,421 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:06,427 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(746): ClusterId : 86152321-5c1e-4b52-a667-0afbaf8c63e9 2024-11-20T15:05:06,427 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(746): ClusterId : 86152321-5c1e-4b52-a667-0afbaf8c63e9 2024-11-20T15:05:06,427 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(746): ClusterId : 86152321-5c1e-4b52-a667-0afbaf8c63e9 2024-11-20T15:05:06,429 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T15:05:06,429 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T15:05:06,429 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T15:05:06,430 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T15:05:06,436 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T15:05:06,441 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a15ecfd95f4,39091,1732115104505 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T15:05:06,461 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T15:05:06,461 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T15:05:06,461 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T15:05:06,461 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T15:05:06,461 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T15:05:06,461 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T15:05:06,462 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T15:05:06,462 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T15:05:06,462 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T15:05:06,462 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=5, maxPoolSize=5 2024-11-20T15:05:06,462 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a15ecfd95f4:0, corePoolSize=10, maxPoolSize=10 2024-11-20T15:05:06,462 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,463 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T15:05:06,463 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,464 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115136464 2024-11-20T15:05:06,466 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T15:05:06,467 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T15:05:06,468 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:06,469 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T15:05:06,470 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T15:05:06,471 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T15:05:06,471 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T15:05:06,471 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T15:05:06,472 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,475 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:06,475 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T15:05:06,475 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T15:05:06,477 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T15:05:06,477 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T15:05:06,477 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T15:05:06,477 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T15:05:06,477 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T15:05:06,478 DEBUG [RS:0;1a15ecfd95f4:42753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8467e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T15:05:06,478 DEBUG [RS:1;1a15ecfd95f4:39199 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711f8de0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T15:05:06,478 DEBUG [RS:2;1a15ecfd95f4:36179 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d34cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a15ecfd95f4/172.17.0.2:0 2024-11-20T15:05:06,482 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T15:05:06,483 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T15:05:06,485 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732115106484,5,FailOnTimeoutGroup] 2024-11-20T15:05:06,492 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732115106485,5,FailOnTimeoutGroup] 2024-11-20T15:05:06,493 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,493 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T15:05:06,494 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,494 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,495 DEBUG [RS:1;1a15ecfd95f4:39199 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1a15ecfd95f4:39199 2024-11-20T15:05:06,495 DEBUG [RS:2;1a15ecfd95f4:36179 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1a15ecfd95f4:36179 2024-11-20T15:05:06,535 DEBUG [RS:0;1a15ecfd95f4:42753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a15ecfd95f4:42753 2024-11-20T15:05:06,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741831_1007 (size=1321) 2024-11-20T15:05:06,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741831_1007 (size=1321) 2024-11-20T15:05:06,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741831_1007 (size=1321) 2024-11-20T15:05:06,538 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T15:05:06,538 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T15:05:06,538 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T15:05:06,538 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T15:05:06,538 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T15:05:06,538 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T15:05:06,538 DEBUG [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T15:05:06,538 DEBUG [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T15:05:06,538 DEBUG [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T15:05:06,540 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T15:05:06,541 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:06,541 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a15ecfd95f4,39091,1732115104505 with port=39199, startcode=1732115105345 2024-11-20T15:05:06,541 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a15ecfd95f4,39091,1732115104505 with port=42753, startcode=1732115105221 2024-11-20T15:05:06,541 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(2659): reportForDuty to master=1a15ecfd95f4,39091,1732115104505 with port=36179, startcode=1732115105395 2024-11-20T15:05:06,553 DEBUG [RS:2;1a15ecfd95f4:36179 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T15:05:06,553 DEBUG [RS:0;1a15ecfd95f4:42753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T15:05:06,553 DEBUG [RS:1;1a15ecfd95f4:39199 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T15:05:06,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741832_1008 (size=32) 2024-11-20T15:05:06,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741832_1008 (size=32) 2024-11-20T15:05:06,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741832_1008 (size=32) 2024-11-20T15:05:06,559 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:06,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T15:05:06,564 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T15:05:06,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:06,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:06,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T15:05:06,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T15:05:06,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:06,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:06,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T15:05:06,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T15:05:06,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:06,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:06,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T15:05:06,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T15:05:06,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:06,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:06,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T15:05:06,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740 2024-11-20T15:05:06,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740 2024-11-20T15:05:06,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T15:05:06,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T15:05:06,591 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57403, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T15:05:06,591 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T15:05:06,591 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34057, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T15:05:06,591 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T15:05:06,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T15:05:06,598 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:06,599 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59716454, jitterRate=-0.11015549302101135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T15:05:06,599 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:06,602 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] master.ServerManager(517): Registering regionserver=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:06,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732115106559Initializing all the Stores at 1732115106561 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115106561Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115106561Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115106561Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115106561Cleaning up temporary data from old regions at 1732115106590 (+29 ms)Region opened successfully at 1732115106602 (+12 ms) 2024-11-20T15:05:06,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T15:05:06,603 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T15:05:06,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T15:05:06,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T15:05:06,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T15:05:06,604 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T15:05:06,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732115106603Disabling compacts and flushes for region at 1732115106603Disabling writes for close at 1732115106603Writing region close event to WAL at 1732115106604 (+1 ms)Closed at 1732115106604 2024-11-20T15:05:06,607 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:06,608 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T15:05:06,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T15:05:06,615 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:06,616 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] master.ServerManager(517): Registering regionserver=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:06,619 DEBUG [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:06,619 DEBUG [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32953 2024-11-20T15:05:06,619 DEBUG [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T15:05:06,621 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:06,621 DEBUG [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:06,621 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] master.ServerManager(517): Registering regionserver=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:06,621 DEBUG [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32953 2024-11-20T15:05:06,621 DEBUG [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T15:05:06,625 DEBUG [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:06,625 DEBUG [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32953 2024-11-20T15:05:06,625 DEBUG [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T15:05:06,627 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:06,631 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T15:05:06,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T15:05:06,681 DEBUG [RS:2;1a15ecfd95f4:36179 {}] zookeeper.ZKUtil(111): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:06,681 WARN [RS:2;1a15ecfd95f4:36179 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T15:05:06,681 DEBUG [RS:0;1a15ecfd95f4:42753 {}] zookeeper.ZKUtil(111): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:06,681 INFO [RS:2;1a15ecfd95f4:36179 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T15:05:06,681 DEBUG [RS:1;1a15ecfd95f4:39199 {}] zookeeper.ZKUtil(111): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:06,681 WARN [RS:0;1a15ecfd95f4:42753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T15:05:06,681 WARN [RS:1;1a15ecfd95f4:39199 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T15:05:06,682 INFO [RS:0;1a15ecfd95f4:42753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T15:05:06,682 INFO [RS:1;1a15ecfd95f4:39199 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T15:05:06,682 DEBUG [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:06,682 DEBUG [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:06,682 DEBUG [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:06,683 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a15ecfd95f4,42753,1732115105221] 2024-11-20T15:05:06,683 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a15ecfd95f4,39199,1732115105345] 2024-11-20T15:05:06,683 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a15ecfd95f4,36179,1732115105395] 2024-11-20T15:05:06,707 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T15:05:06,707 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T15:05:06,707 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T15:05:06,722 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T15:05:06,722 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T15:05:06,723 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T15:05:06,728 INFO [RS:2;1a15ecfd95f4:36179 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T15:05:06,728 INFO [RS:0;1a15ecfd95f4:42753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T15:05:06,728 INFO [RS:1;1a15ecfd95f4:39199 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T15:05:06,728 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,729 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,729 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,730 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T15:05:06,730 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T15:05:06,730 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T15:05:06,736 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T15:05:06,736 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T15:05:06,736 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T15:05:06,737 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,737 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,737 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,738 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,738 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T15:05:06,739 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T15:05:06,739 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=2, maxPoolSize=2 2024-11-20T15:05:06,739 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T15:05:06,739 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,739 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,740 DEBUG [RS:2;1a15ecfd95f4:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T15:05:06,740 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,740 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,740 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,740 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a15ecfd95f4:0, corePoolSize=1, maxPoolSize=1 2024-11-20T15:05:06,740 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T15:05:06,740 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T15:05:06,740 DEBUG [RS:0;1a15ecfd95f4:42753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T15:05:06,740 DEBUG [RS:1;1a15ecfd95f4:39199 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a15ecfd95f4:0, corePoolSize=3, maxPoolSize=3 2024-11-20T15:05:06,741 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,742 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,742 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,742 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,742 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,742 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,36179,1732115105395-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T15:05:06,743 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39199,1732115105345-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,744 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,42753,1732115105221-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T15:05:06,768 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T15:05:06,768 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T15:05:06,768 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T15:05:06,771 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39199,1732115105345-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,771 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,36179,1732115105395-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,771 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,42753,1732115105221-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,771 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,771 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,771 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,772 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.Replication(171): 1a15ecfd95f4,42753,1732115105221 started 2024-11-20T15:05:06,772 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.Replication(171): 1a15ecfd95f4,39199,1732115105345 started 2024-11-20T15:05:06,772 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.Replication(171): 1a15ecfd95f4,36179,1732115105395 started 2024-11-20T15:05:06,782 WARN [1a15ecfd95f4:39091 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T15:05:06,795 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,795 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,795 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:06,795 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(1482): Serving as 1a15ecfd95f4,42753,1732115105221, RpcServer on 1a15ecfd95f4/172.17.0.2:42753, sessionid=0x1015a10928d0001 2024-11-20T15:05:06,795 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(1482): Serving as 1a15ecfd95f4,39199,1732115105345, RpcServer on 1a15ecfd95f4/172.17.0.2:39199, sessionid=0x1015a10928d0002 2024-11-20T15:05:06,795 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(1482): Serving as 1a15ecfd95f4,36179,1732115105395, RpcServer on 1a15ecfd95f4/172.17.0.2:36179, sessionid=0x1015a10928d0003 2024-11-20T15:05:06,796 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T15:05:06,796 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T15:05:06,796 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T15:05:06,796 DEBUG [RS:0;1a15ecfd95f4:42753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:06,796 DEBUG [RS:2;1a15ecfd95f4:36179 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:06,796 DEBUG [RS:1;1a15ecfd95f4:39199 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:06,797 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,42753,1732115105221' 2024-11-20T15:05:06,797 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,36179,1732115105395' 2024-11-20T15:05:06,797 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,39199,1732115105345' 2024-11-20T15:05:06,797 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T15:05:06,797 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T15:05:06,797 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T15:05:06,798 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T15:05:06,798 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T15:05:06,798 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T15:05:06,798 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T15:05:06,798 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T15:05:06,798 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T15:05:06,798 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T15:05:06,798 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T15:05:06,799 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T15:05:06,799 DEBUG [RS:1;1a15ecfd95f4:39199 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:06,799 DEBUG [RS:2;1a15ecfd95f4:36179 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:06,799 DEBUG [RS:0;1a15ecfd95f4:42753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:06,799 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,39199,1732115105345' 2024-11-20T15:05:06,799 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,36179,1732115105395' 2024-11-20T15:05:06,799 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a15ecfd95f4,42753,1732115105221' 2024-11-20T15:05:06,799 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T15:05:06,799 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T15:05:06,799 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T15:05:06,799 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T15:05:06,800 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T15:05:06,800 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T15:05:06,800 DEBUG [RS:2;1a15ecfd95f4:36179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T15:05:06,800 DEBUG [RS:0;1a15ecfd95f4:42753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T15:05:06,800 DEBUG [RS:1;1a15ecfd95f4:39199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T15:05:06,800 INFO [RS:0;1a15ecfd95f4:42753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T15:05:06,800 INFO [RS:2;1a15ecfd95f4:36179 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T15:05:06,800 INFO [RS:1;1a15ecfd95f4:39199 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T15:05:06,800 INFO [RS:0;1a15ecfd95f4:42753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T15:05:06,800 INFO [RS:2;1a15ecfd95f4:36179 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T15:05:06,800 INFO [RS:1;1a15ecfd95f4:39199 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T15:05:06,906 INFO [RS:0;1a15ecfd95f4:42753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T15:05:06,906 INFO [RS:1;1a15ecfd95f4:39199 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T15:05:06,906 INFO [RS:2;1a15ecfd95f4:36179 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T15:05:06,909 INFO [RS:0;1a15ecfd95f4:42753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a15ecfd95f4%2C42753%2C1732115105221, suffix=, logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,42753,1732115105221, archiveDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs, maxLogs=32 2024-11-20T15:05:06,909 INFO [RS:1;1a15ecfd95f4:39199 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a15ecfd95f4%2C39199%2C1732115105345, suffix=, logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,39199,1732115105345, archiveDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs, maxLogs=32 2024-11-20T15:05:06,909 INFO [RS:2;1a15ecfd95f4:36179 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a15ecfd95f4%2C36179%2C1732115105395, suffix=, logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395, archiveDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs, maxLogs=32 2024-11-20T15:05:06,927 DEBUG [RS:0;1a15ecfd95f4:42753 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,42753,1732115105221/1a15ecfd95f4%2C42753%2C1732115105221.1732115106914, exclude list is [], retry=0 2024-11-20T15:05:06,927 DEBUG [RS:1;1a15ecfd95f4:39199 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,39199,1732115105345/1a15ecfd95f4%2C39199%2C1732115105345.1732115106914, exclude list is [], retry=0 2024-11-20T15:05:06,931 DEBUG [RS:2;1a15ecfd95f4:36179 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, exclude list is [], retry=0 2024-11-20T15:05:06,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34807,DS-6246c99a-c186-42f4-960c-91269a275f0d,DISK] 2024-11-20T15:05:06,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38935,DS-a0b7ff8b-cc8f-4112-bea6-13ca8f08a4a2,DISK] 2024-11-20T15:05:06,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38935,DS-a0b7ff8b-cc8f-4112-bea6-13ca8f08a4a2,DISK] 2024-11-20T15:05:06,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42487,DS-77d67676-1e5f-4a3d-9c70-d4f0caedd6ad,DISK] 2024-11-20T15:05:06,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42487,DS-77d67676-1e5f-4a3d-9c70-d4f0caedd6ad,DISK] 2024-11-20T15:05:06,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34807,DS-6246c99a-c186-42f4-960c-91269a275f0d,DISK] 2024-11-20T15:05:06,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34807,DS-6246c99a-c186-42f4-960c-91269a275f0d,DISK] 2024-11-20T15:05:06,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42487,DS-77d67676-1e5f-4a3d-9c70-d4f0caedd6ad,DISK] 2024-11-20T15:05:06,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38935,DS-a0b7ff8b-cc8f-4112-bea6-13ca8f08a4a2,DISK] 2024-11-20T15:05:06,942 INFO [RS:0;1a15ecfd95f4:42753 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,42753,1732115105221/1a15ecfd95f4%2C42753%2C1732115105221.1732115106914 2024-11-20T15:05:06,942 INFO [RS:1;1a15ecfd95f4:39199 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,39199,1732115105345/1a15ecfd95f4%2C39199%2C1732115105345.1732115106914 2024-11-20T15:05:06,942 INFO [RS:2;1a15ecfd95f4:36179 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914 2024-11-20T15:05:06,943 DEBUG [RS:1;1a15ecfd95f4:39199 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44829:44829),(127.0.0.1/127.0.0.1:42775:42775),(127.0.0.1/127.0.0.1:42085:42085)] 2024-11-20T15:05:06,943 DEBUG [RS:0;1a15ecfd95f4:42753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42775:42775),(127.0.0.1/127.0.0.1:42085:42085),(127.0.0.1/127.0.0.1:44829:44829)] 2024-11-20T15:05:06,943 DEBUG [RS:2;1a15ecfd95f4:36179 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44829:44829),(127.0.0.1/127.0.0.1:42775:42775),(127.0.0.1/127.0.0.1:42085:42085)] 2024-11-20T15:05:07,035 DEBUG [1a15ecfd95f4:39091 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T15:05:07,047 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T15:05:07,053 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T15:05:07,053 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T15:05:07,053 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T15:05:07,053 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T15:05:07,053 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T15:05:07,053 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T15:05:07,053 INFO [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T15:05:07,053 INFO [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T15:05:07,053 INFO [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T15:05:07,053 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T15:05:07,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:07,068 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPENING 2024-11-20T15:05:07,126 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T15:05:07,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:07,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:07,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:07,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:07,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:07,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:07,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:07,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:07,141 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:07,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:07,324 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T15:05:07,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T15:05:07,338 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T15:05:07,338 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T15:05:07,338 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T15:05:07,341 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a15ecfd95f4%2C39199%2C1732115105345.meta, suffix=.meta, logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,39199,1732115105345, archiveDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs, maxLogs=32 2024-11-20T15:05:07,358 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,39199,1732115105345/1a15ecfd95f4%2C39199%2C1732115105345.meta.1732115107344.meta, exclude list is [], retry=0 2024-11-20T15:05:07,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38935,DS-a0b7ff8b-cc8f-4112-bea6-13ca8f08a4a2,DISK] 2024-11-20T15:05:07,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34807,DS-6246c99a-c186-42f4-960c-91269a275f0d,DISK] 2024-11-20T15:05:07,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42487,DS-77d67676-1e5f-4a3d-9c70-d4f0caedd6ad,DISK] 2024-11-20T15:05:07,365 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,39199,1732115105345/1a15ecfd95f4%2C39199%2C1732115105345.meta.1732115107344.meta 2024-11-20T15:05:07,366 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42775:42775),(127.0.0.1/127.0.0.1:44829:44829),(127.0.0.1/127.0.0.1:42085:42085)] 2024-11-20T15:05:07,366 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T15:05:07,367 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T15:05:07,370 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T15:05:07,375 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T15:05:07,378 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T15:05:07,379 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,379 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T15:05:07,379 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T15:05:07,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T15:05:07,384 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T15:05:07,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:07,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:07,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T15:05:07,386 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T15:05:07,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:07,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:07,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T15:05:07,389 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T15:05:07,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:07,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:07,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T15:05:07,391 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T15:05:07,392 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:07,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:07,392 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T15:05:07,394 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740 2024-11-20T15:05:07,396 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740 2024-11-20T15:05:07,399 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T15:05:07,399 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T15:05:07,400 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T15:05:07,402 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T15:05:07,404 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72341776, jitterRate=0.07797646522521973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T15:05:07,404 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T15:05:07,405 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732115107380Writing region info on filesystem at 1732115107380Initializing all the Stores at 1732115107382 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115107382Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115107382Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115107382Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115107382Cleaning up temporary data from old regions at 1732115107399 (+17 ms)Running coprocessor post-open hooks at 1732115107404 (+5 ms)Region opened successfully at 1732115107405 (+1 ms) 2024-11-20T15:05:07,411 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732115107316 2024-11-20T15:05:07,421 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T15:05:07,422 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T15:05:07,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:07,425 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:07,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:07,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:07,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:07,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:07,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:07,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:07,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:07,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:07,452 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:07,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T15:05:07,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 in 309 msec 2024-11-20T15:05:07,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T15:05:07,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 846 msec 2024-11-20T15:05:07,467 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:07,467 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T15:05:07,485 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T15:05:07,486 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a15ecfd95f4,39199,1732115105345, seqNum=-1] 2024-11-20T15:05:07,504 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T15:05:07,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36249, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T15:05:07,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1500 sec 2024-11-20T15:05:07,529 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732115107529, completionTime=-1 2024-11-20T15:05:07,532 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T15:05:07,532 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:07,557 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-20T15:05:07,557 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115167557 2024-11-20T15:05:07,557 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115227557 2024-11-20T15:05:07,557 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-20T15:05:07,558 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:07,565 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39091,1732115104505-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:07,565 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39091,1732115104505-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:07,565 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39091,1732115104505-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:07,567 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a15ecfd95f4:39091, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:07,567 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:07,568 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:07,573 DEBUG [master/1a15ecfd95f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T15:05:07,594 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.100sec 2024-11-20T15:05:07,596 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T15:05:07,598 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T15:05:07,598 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T15:05:07,599 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T15:05:07,599 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T15:05:07,600 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39091,1732115104505-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T15:05:07,600 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39091,1732115104505-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T15:05:07,605 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T15:05:07,605 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T15:05:07,606 INFO [master/1a15ecfd95f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a15ecfd95f4,39091,1732115104505-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T15:05:07,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@220c32e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T15:05:07,640 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T15:05:07,640 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T15:05:07,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1a15ecfd95f4,39091,-1 for getting cluster id 2024-11-20T15:05:07,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T15:05:07,654 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '86152321-5c1e-4b52-a667-0afbaf8c63e9' 2024-11-20T15:05:07,656 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T15:05:07,656 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "86152321-5c1e-4b52-a667-0afbaf8c63e9" 2024-11-20T15:05:07,656 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e095a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T15:05:07,656 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1a15ecfd95f4,39091,-1] 2024-11-20T15:05:07,659 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T15:05:07,661 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:07,662 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T15:05:07,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6878c622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T15:05:07,665 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T15:05:07,672 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1a15ecfd95f4,39199,1732115105345, seqNum=-1] 2024-11-20T15:05:07,672 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T15:05:07,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T15:05:07,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:07,698 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T15:05:07,703 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:07,705 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2114013e 2024-11-20T15:05:07,706 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T15:05:07,708 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T15:05:07,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T15:05:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:07,725 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T15:05:07,745 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:07,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-20T15:05:07,749 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T15:05:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T15:05:07,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741837_1013 (size=358) 2024-11-20T15:05:07,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741837_1013 (size=358) 2024-11-20T15:05:07,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741837_1013 (size=358) 2024-11-20T15:05:07,775 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dda0575ccaa2bf831399fa5e62de1046, NAME => 'test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,775 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 65f62894fb325f2b3b65c5f08886f870, NAME => 'test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,781 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 92fc6039f588e69ef1ddcd4fd69369db, NAME => 'test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741838_1014 (size=45) 2024-11-20T15:05:07,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741838_1014 (size=45) 2024-11-20T15:05:07,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741838_1014 (size=45) 2024-11-20T15:05:07,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 65f62894fb325f2b3b65c5f08886f870, disabling compactions & flushes 2024-11-20T15:05:07,811 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:07,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:07,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. after waiting 0 ms 2024-11-20T15:05:07,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:07,811 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:07,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 65f62894fb325f2b3b65c5f08886f870: Waiting for close lock at 1732115107811Disabling compacts and flushes for region at 1732115107811Disabling writes for close at 1732115107811Writing region close event to WAL at 1732115107811Closed at 1732115107811 2024-11-20T15:05:07,812 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 50775deb03089a468fbaec014df1a140, NAME => 'test,ddd,1732115107710.50775deb03089a468fbaec014df1a140.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741840_1016 (size=42) 2024-11-20T15:05:07,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741839_1015 (size=45) 2024-11-20T15:05:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741840_1016 (size=42) 2024-11-20T15:05:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741839_1015 (size=45) 2024-11-20T15:05:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741840_1016 (size=42) 2024-11-20T15:05:07,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741839_1015 (size=45) 2024-11-20T15:05:07,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing dda0575ccaa2bf831399fa5e62de1046, disabling compactions & flushes 2024-11-20T15:05:07,825 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:07,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:07,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. after waiting 0 ms 2024-11-20T15:05:07,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:07,825 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:07,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for dda0575ccaa2bf831399fa5e62de1046: Waiting for close lock at 1732115107825Disabling compacts and flushes for region at 1732115107825Disabling writes for close at 1732115107825Writing region close event to WAL at 1732115107825Closed at 1732115107825 2024-11-20T15:05:07,826 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 67e17e52ef4d4eb06c5ffca586b82a9f, NAME => 'test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,826 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,826 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 92fc6039f588e69ef1ddcd4fd69369db, disabling compactions & flushes 2024-11-20T15:05:07,826 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:07,826 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:07,826 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. after waiting 0 ms 2024-11-20T15:05:07,826 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:07,827 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:07,827 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 92fc6039f588e69ef1ddcd4fd69369db: Waiting for close lock at 1732115107826Disabling compacts and flushes for region at 1732115107826Disabling writes for close at 1732115107826Writing region close event to WAL at 1732115107826Closed at 1732115107827 (+1 ms) 2024-11-20T15:05:07,827 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 804e74fa77c72dca2dc16c98cdc6f0a1, NAME => 'test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741841_1017 (size=45) 2024-11-20T15:05:07,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741841_1017 (size=45) 2024-11-20T15:05:07,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741841_1017 (size=45) 2024-11-20T15:05:07,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1732115107710.50775deb03089a468fbaec014df1a140.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 50775deb03089a468fbaec014df1a140, disabling compactions & flushes 2024-11-20T15:05:07,835 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:07,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:07,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. after waiting 0 ms 2024-11-20T15:05:07,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:07,835 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:07,835 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 50775deb03089a468fbaec014df1a140: Waiting for close lock at 1732115107835Disabling compacts and flushes for region at 1732115107835Disabling writes for close at 1732115107835Writing region close event to WAL at 1732115107835Closed at 1732115107835 2024-11-20T15:05:07,836 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3185afc44c3f735f4f17ff966a0953af, NAME => 'test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741843_1019 (size=45) 2024-11-20T15:05:07,847 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,847 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 804e74fa77c72dca2dc16c98cdc6f0a1, disabling compactions & flushes 2024-11-20T15:05:07,847 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:07,847 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:07,847 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. after waiting 0 ms 2024-11-20T15:05:07,848 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:07,848 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:07,848 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 804e74fa77c72dca2dc16c98cdc6f0a1: Waiting for close lock at 1732115107847Disabling compacts and flushes for region at 1732115107847Disabling writes for close at 1732115107848 (+1 ms)Writing region close event to WAL at 1732115107848Closed at 1732115107848 2024-11-20T15:05:07,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741842_1018 (size=45) 2024-11-20T15:05:07,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741842_1018 (size=45) 2024-11-20T15:05:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741843_1019 (size=45) 2024-11-20T15:05:07,849 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 37573db7e6f70ad7de5e799e1f1056e6, NAME => 'test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741842_1018 (size=45) 2024-11-20T15:05:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741843_1019 (size=45) 2024-11-20T15:05:07,851 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,851 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 67e17e52ef4d4eb06c5ffca586b82a9f, disabling compactions & flushes 2024-11-20T15:05:07,851 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:07,851 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:07,851 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. after waiting 0 ms 2024-11-20T15:05:07,851 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:07,851 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:07,852 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 67e17e52ef4d4eb06c5ffca586b82a9f: Waiting for close lock at 1732115107851Disabling compacts and flushes for region at 1732115107851Disabling writes for close at 1732115107851Writing region close event to WAL at 1732115107851Closed at 1732115107851 2024-11-20T15:05:07,853 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => edc47ba2cc61e109ae6ff5bc44000143, NAME => 'test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T15:05:07,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741844_1020 (size=45) 2024-11-20T15:05:07,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741844_1020 (size=45) 2024-11-20T15:05:07,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741844_1020 (size=45) 2024-11-20T15:05:07,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3185afc44c3f735f4f17ff966a0953af, disabling compactions & flushes 2024-11-20T15:05:07,870 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:07,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:07,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. after waiting 0 ms 2024-11-20T15:05:07,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:07,870 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:07,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3185afc44c3f735f4f17ff966a0953af: Waiting for close lock at 1732115107870Disabling compacts and flushes for region at 1732115107870Disabling writes for close at 1732115107870Writing region close event to WAL at 1732115107870Closed at 1732115107870 2024-11-20T15:05:07,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741845_1021 (size=45) 2024-11-20T15:05:07,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741845_1021 (size=45) 2024-11-20T15:05:07,871 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fff7a80be69d0e9493705d50119cbd2c, NAME => 'test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741845_1021 (size=45) 2024-11-20T15:05:07,872 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 37573db7e6f70ad7de5e799e1f1056e6, disabling compactions & flushes 2024-11-20T15:05:07,873 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:07,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:07,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. after waiting 0 ms 2024-11-20T15:05:07,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:07,873 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:07,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 37573db7e6f70ad7de5e799e1f1056e6: Waiting for close lock at 1732115107873Disabling compacts and flushes for region at 1732115107873Disabling writes for close at 1732115107873Writing region close event to WAL at 1732115107873Closed at 1732115107873 2024-11-20T15:05:07,874 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2201c3d6de103d545e918088a64fbb1d, NAME => 'test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741846_1022 (size=45) 2024-11-20T15:05:07,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741846_1022 (size=45) 2024-11-20T15:05:07,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741846_1022 (size=45) 2024-11-20T15:05:07,881 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,881 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing edc47ba2cc61e109ae6ff5bc44000143, disabling compactions & flushes 2024-11-20T15:05:07,881 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:07,881 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:07,881 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. after waiting 0 ms 2024-11-20T15:05:07,881 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:07,882 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:07,882 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for edc47ba2cc61e109ae6ff5bc44000143: Waiting for close lock at 1732115107881Disabling compacts and flushes for region at 1732115107881Disabling writes for close at 1732115107881Writing region close event to WAL at 1732115107881Closed at 1732115107881 2024-11-20T15:05:07,882 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 69575a66848849b3da0176c19f735873, NAME => 'test,lll,1732115107710.69575a66848849b3da0176c19f735873.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741847_1023 (size=45) 2024-11-20T15:05:07,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741848_1024 (size=45) 2024-11-20T15:05:07,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741848_1024 (size=45) 2024-11-20T15:05:07,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741847_1023 (size=45) 2024-11-20T15:05:07,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741847_1023 (size=45) 2024-11-20T15:05:07,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741848_1024 (size=45) 2024-11-20T15:05:07,891 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,891 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fff7a80be69d0e9493705d50119cbd2c, disabling compactions & flushes 2024-11-20T15:05:07,891 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:07,891 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:07,891 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. after waiting 0 ms 2024-11-20T15:05:07,891 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:07,891 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:07,891 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fff7a80be69d0e9493705d50119cbd2c: Waiting for close lock at 1732115107891Disabling compacts and flushes for region at 1732115107891Disabling writes for close at 1732115107891Writing region close event to WAL at 1732115107891Closed at 1732115107891 2024-11-20T15:05:07,892 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 29a8b1d98f980f11f2986d2cd2aca2ff, NAME => 'test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2201c3d6de103d545e918088a64fbb1d, disabling compactions & flushes 2024-11-20T15:05:07,893 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:07,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:07,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. after waiting 0 ms 2024-11-20T15:05:07,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:07,893 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:07,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2201c3d6de103d545e918088a64fbb1d: Waiting for close lock at 1732115107893Disabling compacts and flushes for region at 1732115107893Disabling writes for close at 1732115107893Writing region close event to WAL at 1732115107893Closed at 1732115107893 2024-11-20T15:05:07,895 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0fd4f4852f3df2562a02cbbf2795ee89, NAME => 'test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741849_1025 (size=45) 2024-11-20T15:05:07,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741849_1025 (size=45) 2024-11-20T15:05:07,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741849_1025 (size=45) 2024-11-20T15:05:07,913 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1732115107710.69575a66848849b3da0176c19f735873.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,913 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 69575a66848849b3da0176c19f735873, disabling compactions & flushes 2024-11-20T15:05:07,913 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:07,913 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:07,913 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732115107710.69575a66848849b3da0176c19f735873. after waiting 0 ms 2024-11-20T15:05:07,913 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:07,913 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:07,913 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 69575a66848849b3da0176c19f735873: Waiting for close lock at 1732115107913Disabling compacts and flushes for region at 1732115107913Disabling writes for close at 1732115107913Writing region close event to WAL at 1732115107913Closed at 1732115107913 2024-11-20T15:05:07,914 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4bcef47d7fdf6bb3fb9febb5d5b742ea, NAME => 'test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741850_1026 (size=45) 2024-11-20T15:05:07,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741850_1026 (size=45) 2024-11-20T15:05:07,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741850_1026 (size=45) 2024-11-20T15:05:07,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 29a8b1d98f980f11f2986d2cd2aca2ff, disabling compactions & flushes 2024-11-20T15:05:07,926 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:07,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:07,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. after waiting 0 ms 2024-11-20T15:05:07,926 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:07,927 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:07,927 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 29a8b1d98f980f11f2986d2cd2aca2ff: Waiting for close lock at 1732115107926Disabling compacts and flushes for region at 1732115107926Disabling writes for close at 1732115107926Writing region close event to WAL at 1732115107927 (+1 ms)Closed at 1732115107927 2024-11-20T15:05:07,928 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7315d7c27f8d82e7df1af9ed20d4c351, NAME => 'test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741851_1027 (size=45) 2024-11-20T15:05:07,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741851_1027 (size=45) 2024-11-20T15:05:07,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741851_1027 (size=45) 2024-11-20T15:05:07,931 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,931 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0fd4f4852f3df2562a02cbbf2795ee89, disabling compactions & flushes 2024-11-20T15:05:07,931 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:07,931 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:07,932 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. after waiting 1 ms 2024-11-20T15:05:07,932 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:07,932 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:07,932 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0fd4f4852f3df2562a02cbbf2795ee89: Waiting for close lock at 1732115107931Disabling compacts and flushes for region at 1732115107931Disabling writes for close at 1732115107932 (+1 ms)Writing region close event to WAL at 1732115107932Closed at 1732115107932 2024-11-20T15:05:07,933 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1142c4fcbc5da26a9d7a32d826bc9c00, NAME => 'test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741852_1028 (size=45) 2024-11-20T15:05:07,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741852_1028 (size=45) 2024-11-20T15:05:07,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741852_1028 (size=45) 2024-11-20T15:05:07,947 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,947 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4bcef47d7fdf6bb3fb9febb5d5b742ea, disabling compactions & flushes 2024-11-20T15:05:07,947 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:07,947 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:07,947 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. after waiting 0 ms 2024-11-20T15:05:07,948 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:07,948 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:07,948 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4bcef47d7fdf6bb3fb9febb5d5b742ea: Waiting for close lock at 1732115107947Disabling compacts and flushes for region at 1732115107947Disabling writes for close at 1732115107947Writing region close event to WAL at 1732115107948 (+1 ms)Closed at 1732115107948 2024-11-20T15:05:07,948 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 26f2170e12c973cd322aaede01781706, NAME => 'test,rrr,1732115107710.26f2170e12c973cd322aaede01781706.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741854_1030 (size=45) 2024-11-20T15:05:07,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741854_1030 (size=45) 2024-11-20T15:05:07,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741854_1030 (size=45) 2024-11-20T15:05:07,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1142c4fcbc5da26a9d7a32d826bc9c00, disabling compactions & flushes 2024-11-20T15:05:07,959 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:07,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:07,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. after waiting 0 ms 2024-11-20T15:05:07,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:07,959 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:07,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1142c4fcbc5da26a9d7a32d826bc9c00: Waiting for close lock at 1732115107959Disabling compacts and flushes for region at 1732115107959Disabling writes for close at 1732115107959Writing region close event to WAL at 1732115107959Closed at 1732115107959 2024-11-20T15:05:07,960 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 600616f3e3ce04088a974ba2eb27ffcc, NAME => 'test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741853_1029 (size=45) 2024-11-20T15:05:07,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741853_1029 (size=45) 2024-11-20T15:05:07,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741853_1029 (size=45) 2024-11-20T15:05:07,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7315d7c27f8d82e7df1af9ed20d4c351, disabling compactions & flushes 2024-11-20T15:05:07,964 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:07,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:07,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. after waiting 0 ms 2024-11-20T15:05:07,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:07,964 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:07,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7315d7c27f8d82e7df1af9ed20d4c351: Waiting for close lock at 1732115107964Disabling compacts and flushes for region at 1732115107964Disabling writes for close at 1732115107964Writing region close event to WAL at 1732115107964Closed at 1732115107964 2024-11-20T15:05:07,965 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 730ee1d6e6dcb8ce4e182db74bb5e284, NAME => 'test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741855_1031 (size=45) 2024-11-20T15:05:07,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741855_1031 (size=45) 2024-11-20T15:05:07,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741855_1031 (size=45) 2024-11-20T15:05:07,976 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732115107710.26f2170e12c973cd322aaede01781706.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,976 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 26f2170e12c973cd322aaede01781706, disabling compactions & flushes 2024-11-20T15:05:07,976 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:07,976 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:07,976 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. after waiting 0 ms 2024-11-20T15:05:07,976 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:07,976 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:07,976 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 26f2170e12c973cd322aaede01781706: Waiting for close lock at 1732115107976Disabling compacts and flushes for region at 1732115107976Disabling writes for close at 1732115107976Writing region close event to WAL at 1732115107976Closed at 1732115107976 2024-11-20T15:05:07,977 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b955aba2c76f221f15758639fba8cacb, NAME => 'test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741856_1032 (size=45) 2024-11-20T15:05:07,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741856_1032 (size=45) 2024-11-20T15:05:07,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741856_1032 (size=45) 2024-11-20T15:05:07,986 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,986 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 600616f3e3ce04088a974ba2eb27ffcc, disabling compactions & flushes 2024-11-20T15:05:07,986 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:07,986 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:07,986 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. after waiting 0 ms 2024-11-20T15:05:07,986 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:07,986 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:07,986 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 600616f3e3ce04088a974ba2eb27ffcc: Waiting for close lock at 1732115107986Disabling compacts and flushes for region at 1732115107986Disabling writes for close at 1732115107986Writing region close event to WAL at 1732115107986Closed at 1732115107986 2024-11-20T15:05:07,987 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 12dda3ed00bfb1c1662ec93d7e6e2e11, NAME => 'test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:07,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741857_1033 (size=45) 2024-11-20T15:05:07,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741857_1033 (size=45) 2024-11-20T15:05:07,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741857_1033 (size=45) 2024-11-20T15:05:07,991 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:07,991 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 730ee1d6e6dcb8ce4e182db74bb5e284, disabling compactions & flushes 2024-11-20T15:05:07,991 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:07,991 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:07,991 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. after waiting 0 ms 2024-11-20T15:05:07,992 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:07,992 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:07,992 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 730ee1d6e6dcb8ce4e182db74bb5e284: Waiting for close lock at 1732115107991Disabling compacts and flushes for region at 1732115107991Disabling writes for close at 1732115107992 (+1 ms)Writing region close event to WAL at 1732115107992Closed at 1732115107992 2024-11-20T15:05:07,992 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 96b1c73fb3ef1f0a71731c98fed31de8, NAME => 'test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:08,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741858_1034 (size=45) 2024-11-20T15:05:08,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741858_1034 (size=45) 2024-11-20T15:05:08,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741860_1036 (size=45) 2024-11-20T15:05:08,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741858_1034 (size=45) 2024-11-20T15:05:08,012 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,012 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b955aba2c76f221f15758639fba8cacb, disabling compactions & flushes 2024-11-20T15:05:08,013 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:08,013 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:08,013 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. after waiting 0 ms 2024-11-20T15:05:08,013 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:08,013 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:08,013 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b955aba2c76f221f15758639fba8cacb: Waiting for close lock at 1732115108012Disabling compacts and flushes for region at 1732115108012Disabling writes for close at 1732115108013 (+1 ms)Writing region close event to WAL at 1732115108013Closed at 1732115108013 2024-11-20T15:05:08,014 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8816e1f68bf2af371e2eeff20fb184ae, NAME => 'test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:08,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741860_1036 (size=45) 2024-11-20T15:05:08,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741860_1036 (size=45) 2024-11-20T15:05:08,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741859_1035 (size=45) 2024-11-20T15:05:08,016 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741859_1035 (size=45) 2024-11-20T15:05:08,016 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 12dda3ed00bfb1c1662ec93d7e6e2e11, disabling compactions & flushes 2024-11-20T15:05:08,016 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:08,016 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:08,016 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. after waiting 0 ms 2024-11-20T15:05:08,016 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:08,016 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:08,016 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 12dda3ed00bfb1c1662ec93d7e6e2e11: Waiting for close lock at 1732115108016Disabling compacts and flushes for region at 1732115108016Disabling writes for close at 1732115108016Writing region close event to WAL at 1732115108016Closed at 1732115108016 2024-11-20T15:05:08,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741859_1035 (size=45) 2024-11-20T15:05:08,017 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 40fff0ed2a6c7cab41e9aeca535ada94, NAME => 'test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:08,017 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,017 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 96b1c73fb3ef1f0a71731c98fed31de8, disabling compactions & flushes 2024-11-20T15:05:08,017 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:08,017 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:08,017 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. after waiting 0 ms 2024-11-20T15:05:08,017 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:08,018 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:08,018 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 96b1c73fb3ef1f0a71731c98fed31de8: Waiting for close lock at 1732115108017Disabling compacts and flushes for region at 1732115108017Disabling writes for close at 1732115108017Writing region close event to WAL at 1732115108018 (+1 ms)Closed at 1732115108018 2024-11-20T15:05:08,018 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 82aa8da18f9efb24d07c39d806b4c510, NAME => 'test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a 2024-11-20T15:05:08,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741861_1037 (size=45) 2024-11-20T15:05:08,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741861_1037 (size=45) 2024-11-20T15:05:08,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741863_1039 (size=42) 2024-11-20T15:05:08,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741863_1039 (size=42) 2024-11-20T15:05:08,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741861_1037 (size=45) 2024-11-20T15:05:08,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741863_1039 (size=42) 2024-11-20T15:05:08,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 82aa8da18f9efb24d07c39d806b4c510, disabling compactions & flushes 2024-11-20T15:05:08,038 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:08,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:08,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. after waiting 0 ms 2024-11-20T15:05:08,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:08,038 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:08,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 82aa8da18f9efb24d07c39d806b4c510: Waiting for close lock at 1732115108038Disabling compacts and flushes for region at 1732115108038Disabling writes for close at 1732115108038Writing region close event to WAL at 1732115108038Closed at 1732115108038 2024-11-20T15:05:08,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741862_1038 (size=45) 2024-11-20T15:05:08,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741862_1038 (size=45) 2024-11-20T15:05:08,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741862_1038 (size=45) 2024-11-20T15:05:08,044 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,044 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8816e1f68bf2af371e2eeff20fb184ae, disabling compactions & flushes 2024-11-20T15:05:08,044 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:08,044 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:08,044 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. after waiting 0 ms 2024-11-20T15:05:08,044 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:08,044 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:08,044 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8816e1f68bf2af371e2eeff20fb184ae: Waiting for close lock at 1732115108044Disabling compacts and flushes for region at 1732115108044Disabling writes for close at 1732115108044Writing region close event to WAL at 1732115108044Closed at 1732115108044 2024-11-20T15:05:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T15:05:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T15:05:08,444 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,444 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 40fff0ed2a6c7cab41e9aeca535ada94, disabling compactions & flushes 2024-11-20T15:05:08,444 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:08,444 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:08,444 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. after waiting 0 ms 2024-11-20T15:05:08,444 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:08,445 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:08,445 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 40fff0ed2a6c7cab41e9aeca535ada94: Waiting for close lock at 1732115108444Disabling compacts and flushes for region at 1732115108444Disabling writes for close at 1732115108444Writing region close event to WAL at 1732115108445 (+1 ms)Closed at 1732115108445 2024-11-20T15:05:08,448 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T15:05:08,456 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,456 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,457 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,457 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732115107710.50775deb03089a468fbaec014df1a140.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,457 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,458 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,458 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,458 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,458 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,459 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,459 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,459 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732115107710.69575a66848849b3da0176c19f735873.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,459 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732115107710.26f2170e12c973cd322aaede01781706.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,461 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,461 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,461 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,461 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,461 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,462 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,462 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732115108448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732115108448"}]},"ts":"1732115108448"} 2024-11-20T15:05:08,500 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-20T15:05:08,503 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T15:05:08,505 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732115108503"}]},"ts":"1732115108503"} 2024-11-20T15:05:08,510 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-20T15:05:08,511 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T15:05:08,528 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T15:05:08,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T15:05:08,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T15:05:08,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T15:05:08,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T15:05:08,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T15:05:08,529 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T15:05:08,529 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T15:05:08,529 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T15:05:08,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T15:05:08,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dda0575ccaa2bf831399fa5e62de1046, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65f62894fb325f2b3b65c5f08886f870, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=50775deb03089a468fbaec014df1a140, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=804e74fa77c72dca2dc16c98cdc6f0a1, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3185afc44c3f735f4f17ff966a0953af, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37573db7e6f70ad7de5e799e1f1056e6, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fff7a80be69d0e9493705d50119cbd2c, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2201c3d6de103d545e918088a64fbb1d, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=29a8b1d98f980f11f2986d2cd2aca2ff, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0fd4f4852f3df2562a02cbbf2795ee89, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7315d7c27f8d82e7df1af9ed20d4c351, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1142c4fcbc5da26a9d7a32d826bc9c00, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=600616f3e3ce04088a974ba2eb27ffcc, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=730ee1d6e6dcb8ce4e182db74bb5e284, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12dda3ed00bfb1c1662ec93d7e6e2e11, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=96b1c73fb3ef1f0a71731c98fed31de8, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8816e1f68bf2af371e2eeff20fb184ae, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40fff0ed2a6c7cab41e9aeca535ada94, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN}] 2024-11-20T15:05:08,539 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:08,541 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:08,542 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40fff0ed2a6c7cab41e9aeca535ada94, ASSIGN 2024-11-20T15:05:08,543 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=40fff0ed2a6c7cab41e9aeca535ada94, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,545 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8816e1f68bf2af371e2eeff20fb184ae, ASSIGN 2024-11-20T15:05:08,546 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8816e1f68bf2af371e2eeff20fb184ae, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,547 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=96b1c73fb3ef1f0a71731c98fed31de8, ASSIGN 2024-11-20T15:05:08,548 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=96b1c73fb3ef1f0a71731c98fed31de8, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,550 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12dda3ed00bfb1c1662ec93d7e6e2e11, ASSIGN 2024-11-20T15:05:08,551 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12dda3ed00bfb1c1662ec93d7e6e2e11, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,552 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:08,554 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:08,555 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=730ee1d6e6dcb8ce4e182db74bb5e284, ASSIGN 2024-11-20T15:05:08,556 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=730ee1d6e6dcb8ce4e182db74bb5e284, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,558 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=600616f3e3ce04088a974ba2eb27ffcc, ASSIGN 2024-11-20T15:05:08,559 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=600616f3e3ce04088a974ba2eb27ffcc, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,560 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:08,561 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:08,563 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1142c4fcbc5da26a9d7a32d826bc9c00, ASSIGN 2024-11-20T15:05:08,565 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1142c4fcbc5da26a9d7a32d826bc9c00, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,566 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7315d7c27f8d82e7df1af9ed20d4c351, ASSIGN 2024-11-20T15:05:08,568 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7315d7c27f8d82e7df1af9ed20d4c351, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,569 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:08,571 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:08,572 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0fd4f4852f3df2562a02cbbf2795ee89, ASSIGN 2024-11-20T15:05:08,573 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0fd4f4852f3df2562a02cbbf2795ee89, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,575 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=29a8b1d98f980f11f2986d2cd2aca2ff, ASSIGN 2024-11-20T15:05:08,576 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=29a8b1d98f980f11f2986d2cd2aca2ff, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,578 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:08,579 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:08,581 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2201c3d6de103d545e918088a64fbb1d, ASSIGN 2024-11-20T15:05:08,582 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2201c3d6de103d545e918088a64fbb1d, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,584 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fff7a80be69d0e9493705d50119cbd2c, ASSIGN 2024-11-20T15:05:08,585 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fff7a80be69d0e9493705d50119cbd2c, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,586 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:08,588 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:08,589 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37573db7e6f70ad7de5e799e1f1056e6, ASSIGN 2024-11-20T15:05:08,591 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=37573db7e6f70ad7de5e799e1f1056e6, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,593 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3185afc44c3f735f4f17ff966a0953af, ASSIGN 2024-11-20T15:05:08,594 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3185afc44c3f735f4f17ff966a0953af, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,596 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=804e74fa77c72dca2dc16c98cdc6f0a1, ASSIGN 2024-11-20T15:05:08,597 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=804e74fa77c72dca2dc16c98cdc6f0a1, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,598 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:08,600 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:08,601 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=50775deb03089a468fbaec014df1a140, ASSIGN 2024-11-20T15:05:08,603 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=50775deb03089a468fbaec014df1a140, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,604 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:08,606 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:08,607 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65f62894fb325f2b3b65c5f08886f870, ASSIGN 2024-11-20T15:05:08,609 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=65f62894fb325f2b3b65c5f08886f870, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,42753,1732115105221; forceNewPlan=false, retain=false 2024-11-20T15:05:08,610 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dda0575ccaa2bf831399fa5e62de1046, ASSIGN 2024-11-20T15:05:08,611 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dda0575ccaa2bf831399fa5e62de1046, ASSIGN; state=OFFLINE, location=1a15ecfd95f4,39199,1732115105345; forceNewPlan=false, retain=false 2024-11-20T15:05:08,698 INFO [1a15ecfd95f4:39091 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-20T15:05:08,699 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,701 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,701 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,702 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=fff7a80be69d0e9493705d50119cbd2c, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8816e1f68bf2af371e2eeff20fb184ae, ASSIGN because future has completed 2024-11-20T15:05:08,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3185afc44c3f735f4f17ff966a0953af, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8816e1f68bf2af371e2eeff20fb184ae, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12dda3ed00bfb1c1662ec93d7e6e2e11, ASSIGN because future has completed 2024-11-20T15:05:08,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=600616f3e3ce04088a974ba2eb27ffcc, ASSIGN because future has completed 2024-11-20T15:05:08,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1142c4fcbc5da26a9d7a32d826bc9c00, ASSIGN because future has completed 2024-11-20T15:05:08,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=29a8b1d98f980f11f2986d2cd2aca2ff, ASSIGN because future has completed 2024-11-20T15:05:08,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29a8b1d98f980f11f2986d2cd2aca2ff, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fff7a80be69d0e9493705d50119cbd2c, ASSIGN because future has completed 2024-11-20T15:05:08,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3185afc44c3f735f4f17ff966a0953af, ASSIGN because future has completed 2024-11-20T15:05:08,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3185afc44c3f735f4f17ff966a0953af, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure fff7a80be69d0e9493705d50119cbd2c, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1142c4fcbc5da26a9d7a32d826bc9c00, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 600616f3e3ce04088a974ba2eb27ffcc, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12dda3ed00bfb1c1662ec93d7e6e2e11, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,746 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=50775deb03089a468fbaec014df1a140, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,746 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=65f62894fb325f2b3b65c5f08886f870, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,747 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,748 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,749 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=50775deb03089a468fbaec014df1a140, ASSIGN because future has completed 2024-11-20T15:05:08,749 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50775deb03089a468fbaec014df1a140, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=65f62894fb325f2b3b65c5f08886f870, ASSIGN because future has completed 2024-11-20T15:05:08,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=40fff0ed2a6c7cab41e9aeca535ada94, ASSIGN because future has completed 2024-11-20T15:05:08,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=96b1c73fb3ef1f0a71731c98fed31de8, ASSIGN because future has completed 2024-11-20T15:05:08,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=730ee1d6e6dcb8ce4e182db74bb5e284, ASSIGN because future has completed 2024-11-20T15:05:08,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 730ee1d6e6dcb8ce4e182db74bb5e284, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7315d7c27f8d82e7df1af9ed20d4c351, ASSIGN because future has completed 2024-11-20T15:05:08,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7315d7c27f8d82e7df1af9ed20d4c351, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96b1c73fb3ef1f0a71731c98fed31de8, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40fff0ed2a6c7cab41e9aeca535ada94, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65f62894fb325f2b3b65c5f08886f870, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:08,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=2201c3d6de103d545e918088a64fbb1d, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dda0575ccaa2bf831399fa5e62de1046, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0fd4f4852f3df2562a02cbbf2795ee89, ASSIGN because future has completed 2024-11-20T15:05:08,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:08,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fd4f4852f3df2562a02cbbf2795ee89, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2201c3d6de103d545e918088a64fbb1d, ASSIGN because future has completed 2024-11-20T15:05:08,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=37573db7e6f70ad7de5e799e1f1056e6, ASSIGN because future has completed 2024-11-20T15:05:08,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=804e74fa77c72dca2dc16c98cdc6f0a1, ASSIGN because future has completed 2024-11-20T15:05:08,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 804e74fa77c72dca2dc16c98cdc6f0a1, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dda0575ccaa2bf831399fa5e62de1046, ASSIGN because future has completed 2024-11-20T15:05:08,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN because future has completed 2024-11-20T15:05:08,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:08,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dda0575ccaa2bf831399fa5e62de1046, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37573db7e6f70ad7de5e799e1f1056e6, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201c3d6de103d545e918088a64fbb1d, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:08,815 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:08,816 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=26f2170e12c973cd322aaede01781706, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:08,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:08,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=69575a66848849b3da0176c19f735873, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:08,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN because future has completed 2024-11-20T15:05:08,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:08,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:08,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN because future has completed 2024-11-20T15:05:08,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN because future has completed 2024-11-20T15:05:08,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN because future has completed 2024-11-20T15:05:08,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN because future has completed 2024-11-20T15:05:08,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:08,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:08,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:08,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:08,848 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:08,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:08,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN because future has completed 2024-11-20T15:05:08,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67e17e52ef4d4eb06c5ffca586b82a9f, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:08,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN because future has completed 2024-11-20T15:05:08,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:08,861 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T15:05:08,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59559, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T15:05:08,868 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:08,868 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 12dda3ed00bfb1c1662ec93d7e6e2e11, NAME => 'test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-20T15:05:08,869 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,869 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,869 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,869 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,871 INFO [StoreOpener-12dda3ed00bfb1c1662ec93d7e6e2e11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,873 INFO [StoreOpener-12dda3ed00bfb1c1662ec93d7e6e2e11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12dda3ed00bfb1c1662ec93d7e6e2e11 columnFamilyName family 2024-11-20T15:05:08,873 DEBUG [StoreOpener-12dda3ed00bfb1c1662ec93d7e6e2e11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,873 INFO [StoreOpener-12dda3ed00bfb1c1662ec93d7e6e2e11-1 {}] regionserver.HStore(327): Store=12dda3ed00bfb1c1662ec93d7e6e2e11/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,874 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,875 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,875 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,876 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,876 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,877 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,881 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/12dda3ed00bfb1c1662ec93d7e6e2e11/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:08,881 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 12dda3ed00bfb1c1662ec93d7e6e2e11; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71914498, jitterRate=0.07160952687263489}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:08,882 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:08,883 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 12dda3ed00bfb1c1662ec93d7e6e2e11: Running coprocessor pre-open hook at 1732115108869Writing region info on filesystem at 1732115108869Initializing all the Stores at 1732115108870 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108871 (+1 ms)Cleaning up temporary data from old regions at 1732115108876 (+5 ms)Running coprocessor post-open hooks at 1732115108882 (+6 ms)Region opened successfully at 1732115108883 (+1 ms) 2024-11-20T15:05:08,885 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11., pid=37, masterSystemTime=1732115108861 2024-11-20T15:05:08,888 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:08,888 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:08,888 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:08,888 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 29a8b1d98f980f11f2986d2cd2aca2ff, NAME => 'test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-20T15:05:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T15:05:08,889 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,889 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,889 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,889 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,891 INFO [StoreOpener-29a8b1d98f980f11f2986d2cd2aca2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12dda3ed00bfb1c1662ec93d7e6e2e11, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:08,894 INFO [StoreOpener-29a8b1d98f980f11f2986d2cd2aca2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29a8b1d98f980f11f2986d2cd2aca2ff columnFamilyName family 2024-11-20T15:05:08,894 DEBUG [StoreOpener-29a8b1d98f980f11f2986d2cd2aca2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,895 INFO [StoreOpener-29a8b1d98f980f11f2986d2cd2aca2ff-1 {}] regionserver.HStore(327): Store=29a8b1d98f980f11f2986d2cd2aca2ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,895 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,897 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,898 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=26 2024-11-20T15:05:08,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 12dda3ed00bfb1c1662ec93d7e6e2e11, server=1a15ecfd95f4,42753,1732115105221 in 153 msec 2024-11-20T15:05:08,898 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,898 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,901 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=12dda3ed00bfb1c1662ec93d7e6e2e11, ASSIGN in 367 msec 2024-11-20T15:05:08,904 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/29a8b1d98f980f11f2986d2cd2aca2ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:08,905 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 29a8b1d98f980f11f2986d2cd2aca2ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72759283, jitterRate=0.08419780433177948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:08,905 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:08,905 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 29a8b1d98f980f11f2986d2cd2aca2ff: Running coprocessor pre-open hook at 1732115108889Writing region info on filesystem at 1732115108889Initializing all the Stores at 1732115108891 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108891Cleaning up temporary data from old regions at 1732115108898 (+7 ms)Running coprocessor post-open hooks at 1732115108905 (+7 ms)Region opened successfully at 1732115108905 2024-11-20T15:05:08,907 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff., pid=32, masterSystemTime=1732115108861 2024-11-20T15:05:08,909 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:08,909 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:08,910 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:08,910 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 600616f3e3ce04088a974ba2eb27ffcc, NAME => 'test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-20T15:05:08,910 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,910 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,910 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,910 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,910 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,913 INFO [StoreOpener-600616f3e3ce04088a974ba2eb27ffcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29a8b1d98f980f11f2986d2cd2aca2ff, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:08,915 INFO [StoreOpener-600616f3e3ce04088a974ba2eb27ffcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 600616f3e3ce04088a974ba2eb27ffcc columnFamilyName family 2024-11-20T15:05:08,915 DEBUG [StoreOpener-600616f3e3ce04088a974ba2eb27ffcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,916 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:08,916 INFO [StoreOpener-600616f3e3ce04088a974ba2eb27ffcc-1 {}] regionserver.HStore(327): Store=600616f3e3ce04088a974ba2eb27ffcc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,916 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fd4f4852f3df2562a02cbbf2795ee89, NAME => 'test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-20T15:05:08,916 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,917 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,917 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,917 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,917 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,918 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,918 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=17 2024-11-20T15:05:08,919 INFO [StoreOpener-0fd4f4852f3df2562a02cbbf2795ee89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 29a8b1d98f980f11f2986d2cd2aca2ff, server=1a15ecfd95f4,42753,1732115105221 in 201 msec 2024-11-20T15:05:08,920 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,920 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=29a8b1d98f980f11f2986d2cd2aca2ff, ASSIGN in 389 msec 2024-11-20T15:05:08,923 INFO [StoreOpener-0fd4f4852f3df2562a02cbbf2795ee89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fd4f4852f3df2562a02cbbf2795ee89 columnFamilyName family 2024-11-20T15:05:08,923 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,923 DEBUG [StoreOpener-0fd4f4852f3df2562a02cbbf2795ee89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,924 INFO [StoreOpener-0fd4f4852f3df2562a02cbbf2795ee89-1 {}] regionserver.HStore(327): Store=0fd4f4852f3df2562a02cbbf2795ee89/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,924 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,925 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,926 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/600616f3e3ce04088a974ba2eb27ffcc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:08,926 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,926 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,926 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,927 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 600616f3e3ce04088a974ba2eb27ffcc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68395333, jitterRate=0.019169881939888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:08,927 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:08,927 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 600616f3e3ce04088a974ba2eb27ffcc: Running coprocessor pre-open hook at 1732115108910Writing region info on filesystem at 1732115108911 (+1 ms)Initializing all the Stores at 1732115108912 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108912Cleaning up temporary data from old regions at 1732115108920 (+8 ms)Running coprocessor post-open hooks at 1732115108927 (+7 ms)Region opened successfully at 1732115108927 2024-11-20T15:05:08,928 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc., pid=36, masterSystemTime=1732115108861 2024-11-20T15:05:08,930 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,931 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:08,932 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:08,932 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:08,932 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 65f62894fb325f2b3b65c5f08886f870, NAME => 'test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-20T15:05:08,933 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,933 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,933 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,933 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,933 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/0fd4f4852f3df2562a02cbbf2795ee89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:08,934 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 0fd4f4852f3df2562a02cbbf2795ee89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65386816, jitterRate=-0.02566051483154297}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:08,934 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:08,935 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 0fd4f4852f3df2562a02cbbf2795ee89: Running coprocessor pre-open hook at 1732115108917Writing region info on filesystem at 1732115108917Initializing all the Stores at 1732115108918 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108919 (+1 ms)Cleaning up temporary data from old regions at 1732115108926 (+7 ms)Running coprocessor post-open hooks at 1732115108934 (+8 ms)Region opened successfully at 1732115108934 2024-11-20T15:05:08,936 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89., pid=44, masterSystemTime=1732115108910 2024-11-20T15:05:08,937 INFO [StoreOpener-65f62894fb325f2b3b65c5f08886f870-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 600616f3e3ce04088a974ba2eb27ffcc, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:08,939 INFO [StoreOpener-65f62894fb325f2b3b65c5f08886f870-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f62894fb325f2b3b65c5f08886f870 columnFamilyName family 2024-11-20T15:05:08,939 DEBUG [StoreOpener-65f62894fb325f2b3b65c5f08886f870-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,939 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:08,940 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:08,940 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:08,940 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => dda0575ccaa2bf831399fa5e62de1046, NAME => 'test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-20T15:05:08,940 INFO [StoreOpener-65f62894fb325f2b3b65c5f08886f870-1 {}] regionserver.HStore(327): Store=65f62894fb325f2b3b65c5f08886f870/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,940 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,940 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,941 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,941 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,941 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,942 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,942 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,943 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,943 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=23 2024-11-20T15:05:08,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 600616f3e3ce04088a974ba2eb27ffcc, server=1a15ecfd95f4,42753,1732115105221 in 202 msec 2024-11-20T15:05:08,945 INFO [StoreOpener-dda0575ccaa2bf831399fa5e62de1046-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,946 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,947 INFO [StoreOpener-dda0575ccaa2bf831399fa5e62de1046-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dda0575ccaa2bf831399fa5e62de1046 columnFamilyName family 2024-11-20T15:05:08,948 DEBUG [StoreOpener-dda0575ccaa2bf831399fa5e62de1046-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=600616f3e3ce04088a974ba2eb27ffcc, ASSIGN in 413 msec 2024-11-20T15:05:08,948 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T15:05:08,949 INFO [StoreOpener-dda0575ccaa2bf831399fa5e62de1046-1 {}] regionserver.HStore(327): Store=dda0575ccaa2bf831399fa5e62de1046/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,949 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,950 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57515, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T15:05:08,952 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,953 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,953 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fd4f4852f3df2562a02cbbf2795ee89, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:08,955 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/65f62894fb325f2b3b65c5f08886f870/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:08,956 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:08,957 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 65f62894fb325f2b3b65c5f08886f870; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62958484, jitterRate=-0.061845481395721436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:08,957 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:08,957 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => edc47ba2cc61e109ae6ff5bc44000143, NAME => 'test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-20T15:05:08,957 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 65f62894fb325f2b3b65c5f08886f870: Running coprocessor pre-open hook at 1732115108933Writing region info on filesystem at 1732115108933Initializing all the Stores at 1732115108936 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108936Cleaning up temporary data from old regions at 1732115108943 (+7 ms)Running coprocessor post-open hooks at 1732115108957 (+14 ms)Region opened successfully at 1732115108957 2024-11-20T15:05:08,957 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,958 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,958 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,958 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,959 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870., pid=43, masterSystemTime=1732115108861 2024-11-20T15:05:08,960 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,961 INFO [StoreOpener-edc47ba2cc61e109ae6ff5bc44000143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,962 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:08,962 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:08,963 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:08,963 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => fff7a80be69d0e9493705d50119cbd2c, NAME => 'test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-20T15:05:08,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=18 2024-11-20T15:05:08,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fd4f4852f3df2562a02cbbf2795ee89, server=1a15ecfd95f4,39199,1732115105345 in 174 msec 2024-11-20T15:05:08,964 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,964 INFO [StoreOpener-edc47ba2cc61e109ae6ff5bc44000143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edc47ba2cc61e109ae6ff5bc44000143 columnFamilyName family 2024-11-20T15:05:08,964 DEBUG [StoreOpener-edc47ba2cc61e109ae6ff5bc44000143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,964 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,964 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,964 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,964 INFO [StoreOpener-edc47ba2cc61e109ae6ff5bc44000143-1 {}] regionserver.HStore(327): Store=edc47ba2cc61e109ae6ff5bc44000143/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,965 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,966 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,966 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,967 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/dda0575ccaa2bf831399fa5e62de1046/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:08,967 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,967 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,967 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened dda0575ccaa2bf831399fa5e62de1046; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73922804, jitterRate=0.1015356183052063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:08,968 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:08,969 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for dda0575ccaa2bf831399fa5e62de1046: Running coprocessor pre-open hook at 1732115108941Writing region info on filesystem at 1732115108941Initializing all the Stores at 1732115108943 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108943Cleaning up temporary data from old regions at 1732115108953 (+10 ms)Running coprocessor post-open hooks at 1732115108968 (+15 ms)Region opened successfully at 1732115108968 2024-11-20T15:05:08,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0fd4f4852f3df2562a02cbbf2795ee89, ASSIGN in 433 msec 2024-11-20T15:05:08,970 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046., pid=47, masterSystemTime=1732115108910 2024-11-20T15:05:08,971 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,971 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:08,973 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:08,973 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:08,974 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:08,974 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 37573db7e6f70ad7de5e799e1f1056e6, NAME => 'test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-20T15:05:08,974 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,974 INFO [StoreOpener-fff7a80be69d0e9493705d50119cbd2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,974 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,975 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,975 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65f62894fb325f2b3b65c5f08886f870, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:08,977 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:08,978 INFO [StoreOpener-fff7a80be69d0e9493705d50119cbd2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fff7a80be69d0e9493705d50119cbd2c columnFamilyName family 2024-11-20T15:05:08,978 DEBUG [StoreOpener-fff7a80be69d0e9493705d50119cbd2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,979 INFO [StoreOpener-fff7a80be69d0e9493705d50119cbd2c-1 {}] regionserver.HStore(327): Store=fff7a80be69d0e9493705d50119cbd2c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,979 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dda0575ccaa2bf831399fa5e62de1046, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:08,981 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,982 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,982 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,982 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=6 2024-11-20T15:05:08,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 65f62894fb325f2b3b65c5f08886f870, server=1a15ecfd95f4,42753,1732115105221 in 205 msec 2024-11-20T15:05:08,985 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:08,988 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/edc47ba2cc61e109ae6ff5bc44000143/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:08,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=65f62894fb325f2b3b65c5f08886f870, ASSIGN in 455 msec 2024-11-20T15:05:08,989 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened edc47ba2cc61e109ae6ff5bc44000143; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72259538, jitterRate=0.07675102353096008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:08,989 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:08,989 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for edc47ba2cc61e109ae6ff5bc44000143: Running coprocessor pre-open hook at 1732115108958Writing region info on filesystem at 1732115108958Initializing all the Stores at 1732115108960 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108960Cleaning up temporary data from old regions at 1732115108967 (+7 ms)Running coprocessor post-open hooks at 1732115108989 (+22 ms)Region opened successfully at 1732115108989 2024-11-20T15:05:08,990 INFO [StoreOpener-37573db7e6f70ad7de5e799e1f1056e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,991 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143., pid=51, masterSystemTime=1732115108947 2024-11-20T15:05:08,992 INFO [StoreOpener-37573db7e6f70ad7de5e799e1f1056e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37573db7e6f70ad7de5e799e1f1056e6 columnFamilyName family 2024-11-20T15:05:08,992 DEBUG [StoreOpener-37573db7e6f70ad7de5e799e1f1056e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:08,993 INFO [StoreOpener-37573db7e6f70ad7de5e799e1f1056e6-1 {}] regionserver.HStore(327): Store=37573db7e6f70ad7de5e799e1f1056e6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:08,994 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:08,994 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:08,994 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:08,994 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,994 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 26f2170e12c973cd322aaede01781706, NAME => 'test,rrr,1732115107710.26f2170e12c973cd322aaede01781706.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-20T15:05:08,994 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:08,995 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,rrr,1732115107710.26f2170e12c973cd322aaede01781706.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:08,995 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:08,995 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:08,996 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,996 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,997 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,997 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:08,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=5 2024-11-20T15:05:08,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dda0575ccaa2bf831399fa5e62de1046, server=1a15ecfd95f4,39199,1732115105345 in 194 msec 2024-11-20T15:05:08,999 INFO [StoreOpener-26f2170e12c973cd322aaede01781706-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:09,001 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:09,002 INFO [StoreOpener-26f2170e12c973cd322aaede01781706-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26f2170e12c973cd322aaede01781706 columnFamilyName family 2024-11-20T15:05:09,002 DEBUG [StoreOpener-26f2170e12c973cd322aaede01781706-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dda0575ccaa2bf831399fa5e62de1046, ASSIGN in 468 msec 2024-11-20T15:05:09,002 INFO [StoreOpener-26f2170e12c973cd322aaede01781706-1 {}] regionserver.HStore(327): Store=26f2170e12c973cd322aaede01781706/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,003 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:09,004 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:09,004 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/26f2170e12c973cd322aaede01781706 2024-11-20T15:05:09,005 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/26f2170e12c973cd322aaede01781706 2024-11-20T15:05:09,007 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:09,007 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:09,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,36179,1732115105395 because future has completed 2024-11-20T15:05:09,010 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/37573db7e6f70ad7de5e799e1f1056e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,010 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/fff7a80be69d0e9493705d50119cbd2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,011 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:09,011 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened fff7a80be69d0e9493705d50119cbd2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72771762, jitterRate=0.0843837559223175}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,011 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:09,012 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for fff7a80be69d0e9493705d50119cbd2c: Running coprocessor pre-open hook at 1732115108964Writing region info on filesystem at 1732115108964Initializing all the Stores at 1732115108966 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108966Cleaning up temporary data from old regions at 1732115108982 (+16 ms)Running coprocessor post-open hooks at 1732115109011 (+29 ms)Region opened successfully at 1732115109012 (+1 ms) 2024-11-20T15:05:09,013 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c., pid=34, masterSystemTime=1732115108861 2024-11-20T15:05:09,011 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 37573db7e6f70ad7de5e799e1f1056e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63863674, jitterRate=-0.048357099294662476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,013 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:09,014 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 37573db7e6f70ad7de5e799e1f1056e6: Running coprocessor pre-open hook at 1732115108975Writing region info on filesystem at 1732115108975Initializing all the Stores at 1732115108981 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108981Cleaning up temporary data from old regions at 1732115108998 (+17 ms)Running coprocessor post-open hooks at 1732115109013 (+15 ms)Region opened successfully at 1732115109013 2024-11-20T15:05:09,016 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6., pid=48, masterSystemTime=1732115108910 2024-11-20T15:05:09,016 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:09,017 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:09,017 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/26f2170e12c973cd322aaede01781706/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,017 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:09,017 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 50775deb03089a468fbaec014df1a140, NAME => 'test,ddd,1732115107710.50775deb03089a468fbaec014df1a140.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-20T15:05:09,017 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,017 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ddd,1732115107710.50775deb03089a468fbaec014df1a140.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,018 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,018 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=13 2024-11-20T15:05:09,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,36179,1732115105395 in 178 msec 2024-11-20T15:05:09,018 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 26f2170e12c973cd322aaede01781706; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65741629, jitterRate=-0.0203733891248703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,019 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:09,019 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 26f2170e12c973cd322aaede01781706: Running coprocessor pre-open hook at 1732115108995Writing region info on filesystem at 1732115108995Initializing all the Stores at 1732115108997 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115108997Cleaning up temporary data from old regions at 1732115109007 (+10 ms)Running coprocessor post-open hooks at 1732115109019 (+12 ms)Region opened successfully at 1732115109019 2024-11-20T15:05:09,020 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732115107710.26f2170e12c973cd322aaede01781706., pid=54, masterSystemTime=1732115108947 2024-11-20T15:05:09,020 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:09,021 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:09,021 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:09,021 INFO [StoreOpener-50775deb03089a468fbaec014df1a140-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,021 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 804e74fa77c72dca2dc16c98cdc6f0a1, NAME => 'test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-20T15:05:09,021 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,022 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,022 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,022 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN in 488 msec 2024-11-20T15:05:09,024 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:09,024 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:09,025 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:09,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:09,025 INFO [StoreOpener-804e74fa77c72dca2dc16c98cdc6f0a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,025 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 69575a66848849b3da0176c19f735873, NAME => 'test,lll,1732115107710.69575a66848849b3da0176c19f735873.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-20T15:05:09,025 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,025 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,lll,1732115107710.69575a66848849b3da0176c19f735873.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,025 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,025 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,026 INFO [StoreOpener-50775deb03089a468fbaec014df1a140-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50775deb03089a468fbaec014df1a140 columnFamilyName family 2024-11-20T15:05:09,026 DEBUG [StoreOpener-50775deb03089a468fbaec014df1a140-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,027 INFO [StoreOpener-50775deb03089a468fbaec014df1a140-1 {}] regionserver.HStore(327): Store=50775deb03089a468fbaec014df1a140/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,028 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,028 INFO [StoreOpener-804e74fa77c72dca2dc16c98cdc6f0a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 804e74fa77c72dca2dc16c98cdc6f0a1 columnFamilyName family 2024-11-20T15:05:09,028 DEBUG [StoreOpener-804e74fa77c72dca2dc16c98cdc6f0a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37573db7e6f70ad7de5e799e1f1056e6, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:09,029 INFO [StoreOpener-804e74fa77c72dca2dc16c98cdc6f0a1-1 {}] regionserver.HStore(327): Store=804e74fa77c72dca2dc16c98cdc6f0a1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,029 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,030 INFO [StoreOpener-69575a66848849b3da0176c19f735873-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,031 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,031 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,031 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,031 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,032 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,032 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,032 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,032 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=26f2170e12c973cd322aaede01781706, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:09,035 INFO [StoreOpener-69575a66848849b3da0176c19f735873-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69575a66848849b3da0176c19f735873 columnFamilyName family 2024-11-20T15:05:09,035 DEBUG [StoreOpener-69575a66848849b3da0176c19f735873-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,036 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,038 INFO [StoreOpener-69575a66848849b3da0176c19f735873-1 {}] regionserver.HStore(327): Store=69575a66848849b3da0176c19f735873/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,36179,1732115105395 because future has completed 2024-11-20T15:05:09,038 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,039 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,039 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,040 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,041 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,041 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=12 2024-11-20T15:05:09,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 37573db7e6f70ad7de5e799e1f1056e6, server=1a15ecfd95f4,39199,1732115105345 in 233 msec 2024-11-20T15:05:09,044 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=37573db7e6f70ad7de5e799e1f1056e6, ASSIGN in 513 msec 2024-11-20T15:05:09,047 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/804e74fa77c72dca2dc16c98cdc6f0a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,048 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 804e74fa77c72dca2dc16c98cdc6f0a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73262997, jitterRate=0.09170372784137726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,049 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:09,049 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 804e74fa77c72dca2dc16c98cdc6f0a1: Running coprocessor pre-open hook at 1732115109022Writing region info on filesystem at 1732115109022Initializing all the Stores at 1732115109024 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109024Cleaning up temporary data from old regions at 1732115109032 (+8 ms)Running coprocessor post-open hooks at 1732115109049 (+17 ms)Region opened successfully at 1732115109049 2024-11-20T15:05:09,050 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1., pid=45, masterSystemTime=1732115108910 2024-11-20T15:05:09,052 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/50775deb03089a468fbaec014df1a140/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,052 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/69575a66848849b3da0176c19f735873/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,053 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 50775deb03089a468fbaec014df1a140; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61694868, jitterRate=-0.08067482709884644}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,053 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 69575a66848849b3da0176c19f735873; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58831761, jitterRate=-0.12333844602108002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,053 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:09,053 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:09,053 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 50775deb03089a468fbaec014df1a140: Running coprocessor pre-open hook at 1732115109018Writing region info on filesystem at 1732115109018Initializing all the Stores at 1732115109020 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109020Cleaning up temporary data from old regions at 1732115109032 (+12 ms)Running coprocessor post-open hooks at 1732115109053 (+21 ms)Region opened successfully at 1732115109053 2024-11-20T15:05:09,053 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 69575a66848849b3da0176c19f735873: Running coprocessor pre-open hook at 1732115109026Writing region info on filesystem at 1732115109026Initializing all the Stores at 1732115109028 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109028Cleaning up temporary data from old regions at 1732115109041 (+13 ms)Running coprocessor post-open hooks at 1732115109053 (+12 ms)Region opened successfully at 1732115109053 2024-11-20T15:05:09,054 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:09,054 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:09,054 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:09,054 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 40fff0ed2a6c7cab41e9aeca535ada94, NAME => 'test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-20T15:05:09,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=22 2024-11-20T15:05:09,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,36179,1732115105395 in 205 msec 2024-11-20T15:05:09,054 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,054 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,055 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,055 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,055 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732115107710.69575a66848849b3da0176c19f735873., pid=52, masterSystemTime=1732115108947 2024-11-20T15:05:09,055 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732115107710.50775deb03089a468fbaec014df1a140., pid=38, masterSystemTime=1732115108861 2024-11-20T15:05:09,058 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:09,058 INFO [StoreOpener-40fff0ed2a6c7cab41e9aeca535ada94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,058 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:09,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN in 523 msec 2024-11-20T15:05:09,059 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:09,059 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 8816e1f68bf2af371e2eeff20fb184ae, NAME => 'test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-20T15:05:09,060 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,060 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,060 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,060 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,061 INFO [StoreOpener-40fff0ed2a6c7cab41e9aeca535ada94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fff0ed2a6c7cab41e9aeca535ada94 columnFamilyName family 2024-11-20T15:05:09,061 DEBUG [StoreOpener-40fff0ed2a6c7cab41e9aeca535ada94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,062 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:09,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=50775deb03089a468fbaec014df1a140, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:09,062 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:09,062 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:09,062 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 82aa8da18f9efb24d07c39d806b4c510, NAME => 'test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-20T15:05:09,062 INFO [StoreOpener-40fff0ed2a6c7cab41e9aeca535ada94-1 {}] regionserver.HStore(327): Store=40fff0ed2a6c7cab41e9aeca535ada94/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,063 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,063 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,063 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,063 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,063 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,064 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,065 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50775deb03089a468fbaec014df1a140, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:09,066 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,066 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,067 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=69575a66848849b3da0176c19f735873, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:09,069 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,36179,1732115105395 because future has completed 2024-11-20T15:05:09,075 INFO [StoreOpener-8816e1f68bf2af371e2eeff20fb184ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=8 2024-11-20T15:05:09,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 50775deb03089a468fbaec014df1a140, server=1a15ecfd95f4,42753,1732115105221 in 321 msec 2024-11-20T15:05:09,076 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/40fff0ed2a6c7cab41e9aeca535ada94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,078 INFO [StoreOpener-8816e1f68bf2af371e2eeff20fb184ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8816e1f68bf2af371e2eeff20fb184ae columnFamilyName family 2024-11-20T15:05:09,078 DEBUG [StoreOpener-8816e1f68bf2af371e2eeff20fb184ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=50775deb03089a468fbaec014df1a140, ASSIGN in 545 msec 2024-11-20T15:05:09,077 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 40fff0ed2a6c7cab41e9aeca535ada94; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73709930, jitterRate=0.09836354851722717}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,078 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:09,079 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 40fff0ed2a6c7cab41e9aeca535ada94: Running coprocessor pre-open hook at 1732115109055Writing region info on filesystem at 1732115109055Initializing all the Stores at 1732115109057 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109057Cleaning up temporary data from old regions at 1732115109066 (+9 ms)Running coprocessor post-open hooks at 1732115109079 (+13 ms)Region opened successfully at 1732115109079 2024-11-20T15:05:09,079 INFO [StoreOpener-8816e1f68bf2af371e2eeff20fb184ae-1 {}] regionserver.HStore(327): Store=8816e1f68bf2af371e2eeff20fb184ae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,079 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,080 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,081 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94., pid=42, masterSystemTime=1732115108910 2024-11-20T15:05:09,081 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,082 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,082 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,084 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:09,084 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:09,084 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:09,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=16 2024-11-20T15:05:09,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,36179,1732115105395 in 243 msec 2024-11-20T15:05:09,085 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 96b1c73fb3ef1f0a71731c98fed31de8, NAME => 'test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-20T15:05:09,085 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,086 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,086 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,086 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,086 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN in 554 msec 2024-11-20T15:05:09,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:09,091 INFO [StoreOpener-82aa8da18f9efb24d07c39d806b4c510-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40fff0ed2a6c7cab41e9aeca535ada94, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:09,094 INFO [StoreOpener-82aa8da18f9efb24d07c39d806b4c510-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82aa8da18f9efb24d07c39d806b4c510 columnFamilyName family 2024-11-20T15:05:09,094 DEBUG [StoreOpener-82aa8da18f9efb24d07c39d806b4c510-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,094 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:09,095 INFO [StoreOpener-96b1c73fb3ef1f0a71731c98fed31de8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,095 INFO [StoreOpener-82aa8da18f9efb24d07c39d806b4c510-1 {}] regionserver.HStore(327): Store=82aa8da18f9efb24d07c39d806b4c510/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,095 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,097 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,097 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/8816e1f68bf2af371e2eeff20fb184ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,097 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,098 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,098 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 8816e1f68bf2af371e2eeff20fb184ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69742059, jitterRate=0.03923766314983368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,098 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,098 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:09,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 804e74fa77c72dca2dc16c98cdc6f0a1, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:09,098 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 8816e1f68bf2af371e2eeff20fb184ae: Running coprocessor pre-open hook at 1732115109060Writing region info on filesystem at 1732115109061 (+1 ms)Initializing all the Stores at 1732115109062 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109062Cleaning up temporary data from old regions at 1732115109082 (+20 ms)Running coprocessor post-open hooks at 1732115109098 (+16 ms)Region opened successfully at 1732115109098 2024-11-20T15:05:09,099 INFO [StoreOpener-96b1c73fb3ef1f0a71731c98fed31de8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96b1c73fb3ef1f0a71731c98fed31de8 columnFamilyName family 2024-11-20T15:05:09,099 DEBUG [StoreOpener-96b1c73fb3ef1f0a71731c98fed31de8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,100 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,101 INFO [StoreOpener-96b1c73fb3ef1f0a71731c98fed31de8-1 {}] regionserver.HStore(327): Store=96b1c73fb3ef1f0a71731c98fed31de8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,101 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,102 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,103 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,103 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae., pid=31, masterSystemTime=1732115108861 2024-11-20T15:05:09,104 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,104 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,105 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/82aa8da18f9efb24d07c39d806b4c510/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=29 2024-11-20T15:05:09,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 40fff0ed2a6c7cab41e9aeca535ada94, server=1a15ecfd95f4,39199,1732115105345 in 328 msec 2024-11-20T15:05:09,106 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:09,107 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:09,107 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 82aa8da18f9efb24d07c39d806b4c510; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74619998, jitterRate=0.11192461848258972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,107 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:09,107 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 82aa8da18f9efb24d07c39d806b4c510: Running coprocessor pre-open hook at 1732115109063Writing region info on filesystem at 1732115109063Initializing all the Stores at 1732115109065 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109065Cleaning up temporary data from old regions at 1732115109098 (+33 ms)Running coprocessor post-open hooks at 1732115109107 (+9 ms)Region opened successfully at 1732115109107 2024-11-20T15:05:09,107 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,107 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:09,108 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 3185afc44c3f735f4f17ff966a0953af, NAME => 'test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-20T15:05:09,108 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,108 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,108 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510., pid=46, masterSystemTime=1732115108947 2024-11-20T15:05:09,109 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,109 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=40fff0ed2a6c7cab41e9aeca535ada94, ASSIGN in 575 msec 2024-11-20T15:05:09,111 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/96b1c73fb3ef1f0a71731c98fed31de8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,112 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 96b1c73fb3ef1f0a71731c98fed31de8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69917366, jitterRate=0.04184994101524353}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,112 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:09,112 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:09,112 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 96b1c73fb3ef1f0a71731c98fed31de8: Running coprocessor pre-open hook at 1732115109086Writing region info on filesystem at 1732115109086Initializing all the Stores at 1732115109088 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109088Cleaning up temporary data from old regions at 1732115109104 (+16 ms)Running coprocessor post-open hooks at 1732115109112 (+8 ms)Region opened successfully at 1732115109112 2024-11-20T15:05:09,112 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:09,113 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:09,113 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => b955aba2c76f221f15758639fba8cacb, NAME => 'test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-20T15:05:09,113 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8., pid=41, masterSystemTime=1732115108910 2024-11-20T15:05:09,113 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:09,113 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,114 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,114 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,114 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,116 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:09,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8816e1f68bf2af371e2eeff20fb184ae, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:09,117 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:09,117 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:09,117 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 7315d7c27f8d82e7df1af9ed20d4c351, NAME => 'test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-20T15:05:09,117 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,118 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,118 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,118 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,119 INFO [StoreOpener-3185afc44c3f735f4f17ff966a0953af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,119 INFO [StoreOpener-b955aba2c76f221f15758639fba8cacb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,119 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:09,121 INFO [StoreOpener-3185afc44c3f735f4f17ff966a0953af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3185afc44c3f735f4f17ff966a0953af columnFamilyName family 2024-11-20T15:05:09,121 DEBUG [StoreOpener-3185afc44c3f735f4f17ff966a0953af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,121 INFO [StoreOpener-7315d7c27f8d82e7df1af9ed20d4c351-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,123 INFO [StoreOpener-3185afc44c3f735f4f17ff966a0953af-1 {}] regionserver.HStore(327): Store=3185afc44c3f735f4f17ff966a0953af/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96b1c73fb3ef1f0a71731c98fed31de8, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:09,124 INFO [StoreOpener-b955aba2c76f221f15758639fba8cacb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b955aba2c76f221f15758639fba8cacb columnFamilyName family 2024-11-20T15:05:09,124 DEBUG [StoreOpener-b955aba2c76f221f15758639fba8cacb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,124 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,125 INFO [StoreOpener-7315d7c27f8d82e7df1af9ed20d4c351-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7315d7c27f8d82e7df1af9ed20d4c351 columnFamilyName family 2024-11-20T15:05:09,125 DEBUG [StoreOpener-7315d7c27f8d82e7df1af9ed20d4c351-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,126 INFO [StoreOpener-b955aba2c76f221f15758639fba8cacb-1 {}] regionserver.HStore(327): Store=b955aba2c76f221f15758639fba8cacb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,126 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,126 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,126 INFO [StoreOpener-7315d7c27f8d82e7df1af9ed20d4c351-1 {}] regionserver.HStore(327): Store=7315d7c27f8d82e7df1af9ed20d4c351/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,127 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,127 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,127 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,128 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,128 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,128 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,128 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,128 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,129 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,129 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,129 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,129 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-20T15:05:09,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 8816e1f68bf2af371e2eeff20fb184ae, server=1a15ecfd95f4,42753,1732115105221 in 418 msec 2024-11-20T15:05:09,131 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,131 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,131 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8816e1f68bf2af371e2eeff20fb184ae, ASSIGN in 599 msec 2024-11-20T15:05:09,135 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/7315d7c27f8d82e7df1af9ed20d4c351/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,137 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 7315d7c27f8d82e7df1af9ed20d4c351; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70770749, jitterRate=0.054566338658332825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,137 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:09,137 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 7315d7c27f8d82e7df1af9ed20d4c351: Running coprocessor pre-open hook at 1732115109118Writing region info on filesystem at 1732115109118Initializing all the Stores at 1732115109119 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109120 (+1 ms)Cleaning up temporary data from old regions at 1732115109129 (+9 ms)Running coprocessor post-open hooks at 1732115109137 (+8 ms)Region opened successfully at 1732115109137 2024-11-20T15:05:09,138 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351., pid=40, masterSystemTime=1732115108910 2024-11-20T15:05:09,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=27 2024-11-20T15:05:09,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 96b1c73fb3ef1f0a71731c98fed31de8, server=1a15ecfd95f4,39199,1732115105345 in 370 msec 2024-11-20T15:05:09,142 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:09,142 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:09,143 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:09,143 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 730ee1d6e6dcb8ce4e182db74bb5e284, NAME => 'test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-20T15:05:09,143 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,143 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,144 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=test, region=804e74fa77c72dca2dc16c98cdc6f0a1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T15:05:09,144 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,144 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,145 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:09,148 INFO [StoreOpener-730ee1d6e6dcb8ce4e182db74bb5e284-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,149 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/b955aba2c76f221f15758639fba8cacb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7315d7c27f8d82e7df1af9ed20d4c351, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:09,149 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/3185afc44c3f735f4f17ff966a0953af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,150 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 3185afc44c3f735f4f17ff966a0953af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62049649, jitterRate=-0.07538817822933197}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,151 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:09,151 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 3185afc44c3f735f4f17ff966a0953af: Running coprocessor pre-open hook at 1732115109110Writing region info on filesystem at 1732115109110Initializing all the Stores at 1732115109111 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109111Cleaning up temporary data from old regions at 1732115109128 (+17 ms)Running coprocessor post-open hooks at 1732115109151 (+23 ms)Region opened successfully at 1732115109151 2024-11-20T15:05:09,152 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened b955aba2c76f221f15758639fba8cacb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66941046, jitterRate=-0.002500683069229126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=96b1c73fb3ef1f0a71731c98fed31de8, ASSIGN in 617 msec 2024-11-20T15:05:09,152 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:09,152 INFO [StoreOpener-730ee1d6e6dcb8ce4e182db74bb5e284-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 730ee1d6e6dcb8ce4e182db74bb5e284 columnFamilyName family 2024-11-20T15:05:09,152 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for b955aba2c76f221f15758639fba8cacb: Running coprocessor pre-open hook at 1732115109114Writing region info on filesystem at 1732115109114Initializing all the Stores at 1732115109116 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109116Cleaning up temporary data from old regions at 1732115109129 (+13 ms)Running coprocessor post-open hooks at 1732115109152 (+23 ms)Region opened successfully at 1732115109152 2024-11-20T15:05:09,152 DEBUG [StoreOpener-730ee1d6e6dcb8ce4e182db74bb5e284-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,153 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af., pid=33, masterSystemTime=1732115108861 2024-11-20T15:05:09,154 INFO [StoreOpener-730ee1d6e6dcb8ce4e182db74bb5e284-1 {}] regionserver.HStore(327): Store=730ee1d6e6dcb8ce4e182db74bb5e284/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,154 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,155 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb., pid=50, masterSystemTime=1732115108947 2024-11-20T15:05:09,155 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,156 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,156 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:09,156 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:09,156 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:09,157 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1142c4fcbc5da26a9d7a32d826bc9c00, NAME => 'test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-20T15:05:09,157 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,157 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,157 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,157 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,158 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,158 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,160 INFO [StoreOpener-1142c4fcbc5da26a9d7a32d826bc9c00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,161 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:09,161 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:09,162 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:09,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=20 2024-11-20T15:05:09,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 7315d7c27f8d82e7df1af9ed20d4c351, server=1a15ecfd95f4,39199,1732115105345 in 393 msec 2024-11-20T15:05:09,162 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 4bcef47d7fdf6bb3fb9febb5d5b742ea, NAME => 'test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-20T15:05:09,162 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,162 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,162 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,162 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,162 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,163 INFO [StoreOpener-1142c4fcbc5da26a9d7a32d826bc9c00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1142c4fcbc5da26a9d7a32d826bc9c00 columnFamilyName family 2024-11-20T15:05:09,163 DEBUG [StoreOpener-1142c4fcbc5da26a9d7a32d826bc9c00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,164 INFO [StoreOpener-1142c4fcbc5da26a9d7a32d826bc9c00-1 {}] regionserver.HStore(327): Store=1142c4fcbc5da26a9d7a32d826bc9c00/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,164 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,165 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,166 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7315d7c27f8d82e7df1af9ed20d4c351, ASSIGN in 631 msec 2024-11-20T15:05:09,166 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,166 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,167 INFO [StoreOpener-4bcef47d7fdf6bb3fb9febb5d5b742ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,168 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/730ee1d6e6dcb8ce4e182db74bb5e284/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,168 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b955aba2c76f221f15758639fba8cacb, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:09,169 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,169 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 730ee1d6e6dcb8ce4e182db74bb5e284; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73701207, jitterRate=0.09823356568813324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,169 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:09,169 INFO [StoreOpener-4bcef47d7fdf6bb3fb9febb5d5b742ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4bcef47d7fdf6bb3fb9febb5d5b742ea columnFamilyName family 2024-11-20T15:05:09,169 DEBUG [StoreOpener-4bcef47d7fdf6bb3fb9febb5d5b742ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,170 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 730ee1d6e6dcb8ce4e182db74bb5e284: Running coprocessor pre-open hook at 1732115109144Writing region info on filesystem at 1732115109144Initializing all the Stores at 1732115109146 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109146Cleaning up temporary data from old regions at 1732115109158 (+12 ms)Running coprocessor post-open hooks at 1732115109169 (+11 ms)Region opened successfully at 1732115109169 2024-11-20T15:05:09,170 INFO [StoreOpener-4bcef47d7fdf6bb3fb9febb5d5b742ea-1 {}] regionserver.HStore(327): Store=4bcef47d7fdf6bb3fb9febb5d5b742ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,171 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,172 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284., pid=39, masterSystemTime=1732115108910 2024-11-20T15:05:09,172 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,36179,1732115105395 because future has completed 2024-11-20T15:05:09,173 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,174 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,174 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,175 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/1142c4fcbc5da26a9d7a32d826bc9c00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,176 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1142c4fcbc5da26a9d7a32d826bc9c00; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58758052, jitterRate=-0.12443679571151733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,176 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:09,176 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:09,176 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1142c4fcbc5da26a9d7a32d826bc9c00: Running coprocessor pre-open hook at 1732115109158Writing region info on filesystem at 1732115109158Initializing all the Stores at 1732115109159 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109159Cleaning up temporary data from old regions at 1732115109166 (+7 ms)Running coprocessor post-open hooks at 1732115109176 (+10 ms)Region opened successfully at 1732115109176 2024-11-20T15:05:09,177 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:09,177 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:09,177 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00., pid=35, masterSystemTime=1732115108861 2024-11-20T15:05:09,177 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:09,178 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,178 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 2201c3d6de103d545e918088a64fbb1d, NAME => 'test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-20T15:05:09,180 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,180 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,180 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,180 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3185afc44c3f735f4f17ff966a0953af, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:09,181 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:09,182 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:09,182 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:09,183 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/4bcef47d7fdf6bb3fb9febb5d5b742ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 730ee1d6e6dcb8ce4e182db74bb5e284, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:09,185 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 4bcef47d7fdf6bb3fb9febb5d5b742ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64052846, jitterRate=-0.04553821682929993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,185 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:09,185 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 4bcef47d7fdf6bb3fb9febb5d5b742ea: Running coprocessor pre-open hook at 1732115109163Writing region info on filesystem at 1732115109163Initializing all the Stores at 1732115109164 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109164Cleaning up temporary data from old regions at 1732115109174 (+10 ms)Running coprocessor post-open hooks at 1732115109185 (+11 ms)Region opened successfully at 1732115109185 2024-11-20T15:05:09,186 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:09,186 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea., pid=53, masterSystemTime=1732115108947 2024-11-20T15:05:09,188 INFO [StoreOpener-2201c3d6de103d545e918088a64fbb1d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1142c4fcbc5da26a9d7a32d826bc9c00, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:09,190 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:09,190 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:09,190 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:09,190 INFO [StoreOpener-2201c3d6de103d545e918088a64fbb1d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2201c3d6de103d545e918088a64fbb1d columnFamilyName family 2024-11-20T15:05:09,190 DEBUG [StoreOpener-2201c3d6de103d545e918088a64fbb1d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,190 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 67e17e52ef4d4eb06c5ffca586b82a9f, NAME => 'test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-20T15:05:09,191 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,191 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,191 INFO [StoreOpener-2201c3d6de103d545e918088a64fbb1d-1 {}] regionserver.HStore(327): Store=2201c3d6de103d545e918088a64fbb1d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,191 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,191 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,191 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,192 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,193 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,193 INFO [StoreOpener-67e17e52ef4d4eb06c5ffca586b82a9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,193 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,194 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,196 INFO [StoreOpener-67e17e52ef4d4eb06c5ffca586b82a9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67e17e52ef4d4eb06c5ffca586b82a9f columnFamilyName family 2024-11-20T15:05:09,196 DEBUG [StoreOpener-67e17e52ef4d4eb06c5ffca586b82a9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-11-20T15:05:09,196 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,197 INFO [StoreOpener-67e17e52ef4d4eb06c5ffca586b82a9f-1 {}] regionserver.HStore(327): Store=67e17e52ef4d4eb06c5ffca586b82a9f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 730ee1d6e6dcb8ce4e182db74bb5e284, server=1a15ecfd95f4,39199,1732115105345 in 435 msec 2024-11-20T15:05:09,197 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,198 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,199 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,200 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/2201c3d6de103d545e918088a64fbb1d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,200 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,200 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=730ee1d6e6dcb8ce4e182db74bb5e284, ASSIGN in 666 msec 2024-11-20T15:05:09,201 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 2201c3d6de103d545e918088a64fbb1d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70391639, jitterRate=0.04891715943813324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,201 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:09,201 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 2201c3d6de103d545e918088a64fbb1d: Running coprocessor pre-open hook at 1732115109181Writing region info on filesystem at 1732115109181Initializing all the Stores at 1732115109183 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109183Cleaning up temporary data from old regions at 1732115109194 (+11 ms)Running coprocessor post-open hooks at 1732115109201 (+7 ms)Region opened successfully at 1732115109201 2024-11-20T15:05:09,203 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d., pid=49, masterSystemTime=1732115108910 2024-11-20T15:05:09,203 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,206 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:09,206 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:09,207 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/67e17e52ef4d4eb06c5ffca586b82a9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=21 2024-11-20T15:05:09,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 1142c4fcbc5da26a9d7a32d826bc9c00, server=1a15ecfd95f4,42753,1732115105221 in 472 msec 2024-11-20T15:05:09,207 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 67e17e52ef4d4eb06c5ffca586b82a9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64335883, jitterRate=-0.04132063686847687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,208 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:09,208 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 67e17e52ef4d4eb06c5ffca586b82a9f: Running coprocessor pre-open hook at 1732115109191Writing region info on filesystem at 1732115109191Initializing all the Stores at 1732115109192 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109192Cleaning up temporary data from old regions at 1732115109200 (+8 ms)Running coprocessor post-open hooks at 1732115109208 (+8 ms)Region opened successfully at 1732115109208 2024-11-20T15:05:09,209 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f., pid=55, masterSystemTime=1732115108947 2024-11-20T15:05:09,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1142c4fcbc5da26a9d7a32d826bc9c00, ASSIGN in 676 msec 2024-11-20T15:05:09,211 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:09,211 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:09,211 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:09,212 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:09,212 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 92fc6039f588e69ef1ddcd4fd69369db, NAME => 'test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-20T15:05:09,212 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,213 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:09,213 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,213 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201c3d6de103d545e918088a64fbb1d, server=1a15ecfd95f4,39199,1732115105345 because future has completed 2024-11-20T15:05:09,214 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:09,215 INFO [StoreOpener-92fc6039f588e69ef1ddcd4fd69369db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,217 INFO [StoreOpener-92fc6039f588e69ef1ddcd4fd69369db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92fc6039f588e69ef1ddcd4fd69369db columnFamilyName family 2024-11-20T15:05:09,217 DEBUG [StoreOpener-92fc6039f588e69ef1ddcd4fd69369db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:09,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67e17e52ef4d4eb06c5ffca586b82a9f, server=1a15ecfd95f4,36179,1732115105395 because future has completed 2024-11-20T15:05:09,218 INFO [StoreOpener-92fc6039f588e69ef1ddcd4fd69369db-1 {}] regionserver.HStore(327): Store=92fc6039f588e69ef1ddcd4fd69369db/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:09,218 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,219 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,220 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,221 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,221 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=15 2024-11-20T15:05:09,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 2201c3d6de103d545e918088a64fbb1d, server=1a15ecfd95f4,39199,1732115105345 in 409 msec 2024-11-20T15:05:09,224 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2201c3d6de103d545e918088a64fbb1d, ASSIGN in 692 msec 2024-11-20T15:05:09,227 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/92fc6039f588e69ef1ddcd4fd69369db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T15:05:09,228 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 92fc6039f588e69ef1ddcd4fd69369db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67290529, jitterRate=0.0027070194482803345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:09,228 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:09,228 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 92fc6039f588e69ef1ddcd4fd69369db: Running coprocessor pre-open hook at 1732115109213Writing region info on filesystem at 1732115109213Initializing all the Stores at 1732115109214 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115109214Cleaning up temporary data from old regions at 1732115109221 (+7 ms)Running coprocessor post-open hooks at 1732115109228 (+7 ms)Region opened successfully at 1732115109228 2024-11-20T15:05:09,229 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db., pid=56, masterSystemTime=1732115108947 2024-11-20T15:05:09,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-11-20T15:05:09,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 67e17e52ef4d4eb06c5ffca586b82a9f, server=1a15ecfd95f4,36179,1732115105395 in 376 msec 2024-11-20T15:05:09,232 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:09,232 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:09,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN in 700 msec 2024-11-20T15:05:09,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:09,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,36179,1732115105395 because future has completed 2024-11-20T15:05:09,238 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:09,240 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1a15ecfd95f4,36179,1732115105395, table=test, region=92fc6039f588e69ef1ddcd4fd69369db. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T15:05:09,240 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1a15ecfd95f4,36179,1732115105395, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T15:05:09,240 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1a15ecfd95f4,36179,1732115105395, table=test, region=b955aba2c76f221f15758639fba8cacb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T15:05:09,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,36179,1732115105395 because future has completed 2024-11-20T15:05:09,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=7 2024-11-20T15:05:09,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,36179,1732115105395 in 387 msec 2024-11-20T15:05:09,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN in 717 msec 2024-11-20T15:05:09,252 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1a15ecfd95f4,42753,1732115105221, table=test, region=3185afc44c3f735f4f17ff966a0953af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T15:05:09,253 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=test, region=804e74fa77c72dca2dc16c98cdc6f0a1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T15:05:09,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=19 2024-11-20T15:05:09,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,36179,1732115105395 in 411 msec 2024-11-20T15:05:09,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN in 724 msec 2024-11-20T15:05:09,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=11 2024-11-20T15:05:09,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3185afc44c3f735f4f17ff966a0953af, server=1a15ecfd95f4,42753,1732115105221 in 537 msec 2024-11-20T15:05:09,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3185afc44c3f735f4f17ff966a0953af, ASSIGN in 730 msec 2024-11-20T15:05:09,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=25 2024-11-20T15:05:09,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,36179,1732115105395 in 445 msec 2024-11-20T15:05:09,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN in 736 msec 2024-11-20T15:05:09,270 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=82aa8da18f9efb24d07c39d806b4c510, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:09,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,36179,1732115105395 because future has completed 2024-11-20T15:05:09,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=10 2024-11-20T15:05:09,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 804e74fa77c72dca2dc16c98cdc6f0a1, server=1a15ecfd95f4,39199,1732115105345 in 484 msec 2024-11-20T15:05:09,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=804e74fa77c72dca2dc16c98cdc6f0a1, ASSIGN in 745 msec 2024-11-20T15:05:09,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=30 2024-11-20T15:05:09,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,36179,1732115105395 in 484 msec 2024-11-20T15:05:09,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN in 750 msec 2024-11-20T15:05:09,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, openSeqNum=2, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:09,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure fff7a80be69d0e9493705d50119cbd2c, server=1a15ecfd95f4,42753,1732115105221 because future has completed 2024-11-20T15:05:09,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=14 2024-11-20T15:05:09,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure fff7a80be69d0e9493705d50119cbd2c, server=1a15ecfd95f4,42753,1732115105221 in 562 msec 2024-11-20T15:05:09,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=4 2024-11-20T15:05:09,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fff7a80be69d0e9493705d50119cbd2c, ASSIGN in 761 msec 2024-11-20T15:05:09,296 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T15:05:09,297 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732115109296"}]},"ts":"1732115109296"} 2024-11-20T15:05:09,299 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-20T15:05:09,302 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T15:05:09,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5860 sec 2024-11-20T15:05:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T15:05:09,902 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-20T15:05:09,902 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-20T15:05:09,905 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T15:05:09,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-20T15:05:09,926 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T15:05:09,927 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-20T15:05:09,928 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T15:05:09,933 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T15:05:09,956 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T15:05:09,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T15:05:09,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T15:05:09,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T15:05:09,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T15:05:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 92fc6039f588e69ef1ddcd4fd69369db changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 67e17e52ef4d4eb06c5ffca586b82a9f changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region edc47ba2cc61e109ae6ff5bc44000143 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 69575a66848849b3da0176c19f735873 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4bcef47d7fdf6bb3fb9febb5d5b742ea changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 26f2170e12c973cd322aaede01781706 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b955aba2c76f221f15758639fba8cacb changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 82aa8da18f9efb24d07c39d806b4c510 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dda0575ccaa2bf831399fa5e62de1046 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 804e74fa77c72dca2dc16c98cdc6f0a1 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 37573db7e6f70ad7de5e799e1f1056e6 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2201c3d6de103d545e918088a64fbb1d changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0fd4f4852f3df2562a02cbbf2795ee89 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7315d7c27f8d82e7df1af9ed20d4c351 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 730ee1d6e6dcb8ce4e182db74bb5e284 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 96b1c73fb3ef1f0a71731c98fed31de8 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 40fff0ed2a6c7cab41e9aeca535ada94 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 65f62894fb325f2b3b65c5f08886f870 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 50775deb03089a468fbaec014df1a140 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3185afc44c3f735f4f17ff966a0953af changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fff7a80be69d0e9493705d50119cbd2c changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 29a8b1d98f980f11f2986d2cd2aca2ff changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1142c4fcbc5da26a9d7a32d826bc9c00 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 600616f3e3ce04088a974ba2eb27ffcc changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 12dda3ed00bfb1c1662ec93d7e6e2e11 changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8816e1f68bf2af371e2eeff20fb184ae changed from -1.0 to 0.0, refreshing cache 2024-11-20T15:05:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-20T15:05:10,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-20T15:05:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-20T15:05:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T15:05:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(303): server 0 has 8 regions 2024-11-20T15:05:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-20T15:05:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-20T15:05:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T15:05:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T15:05:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T15:05:10,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T15:05:10,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T15:05:10,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T15:05:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-20T15:05:10,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-20T15:05:10,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-20T15:05:10,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 476 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-20T15:05:10,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-20T15:05:10,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.HMaster(2172): balance hri=1588230740, source=1a15ecfd95f4,39199,1732115105345, destination=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:10,534 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:10,537 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=1588230740, regionState=CLOSING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:10,538 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=CLOSING 2024-11-20T15:05:10,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:10,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:10,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:10,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:10,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:10,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:10,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:10,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:10,593 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:10,593 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T15:05:10,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:10,752 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] handler.UnassignRegionHandler(122): Close 1588230740 2024-11-20T15:05:10,752 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T15:05:10,753 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T15:05:10,753 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T15:05:10,753 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T15:05:10,753 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T15:05:10,753 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T15:05:10,759 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-20T15:05:10,842 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/info/3bfb7ea755e747edbf6d73a666ef2234 is 138, key is test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870./info:regioninfo/1732115108971/Put/seqid=0 2024-11-20T15:05:10,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741864_1040 (size=34924) 2024-11-20T15:05:10,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741864_1040 (size=34924) 2024-11-20T15:05:10,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741864_1040 (size=34924) 2024-11-20T15:05:10,856 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/info/3bfb7ea755e747edbf6d73a666ef2234 2024-11-20T15:05:10,915 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/ns/7279affc7b074592951e2e295a779e91 is 43, key is default/ns:d/1732115107510/Put/seqid=0 2024-11-20T15:05:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741865_1041 (size=5153) 2024-11-20T15:05:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741865_1041 (size=5153) 2024-11-20T15:05:10,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741865_1041 (size=5153) 2024-11-20T15:05:10,925 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/ns/7279affc7b074592951e2e295a779e91 2024-11-20T15:05:10,950 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/table/490d71ae8c9d40a085bb108ea1f06219 is 40, key is test/table:state/1732115109296/Put/seqid=0 2024-11-20T15:05:10,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741866_1042 (size=5165) 2024-11-20T15:05:10,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741866_1042 (size=5165) 2024-11-20T15:05:10,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741866_1042 (size=5165) 2024-11-20T15:05:10,959 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/table/490d71ae8c9d40a085bb108ea1f06219 2024-11-20T15:05:10,970 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/info/3bfb7ea755e747edbf6d73a666ef2234 as hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/info/3bfb7ea755e747edbf6d73a666ef2234 2024-11-20T15:05:10,980 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/info/3bfb7ea755e747edbf6d73a666ef2234, entries=260, sequenceid=61, filesize=34.1 K 2024-11-20T15:05:10,984 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/ns/7279affc7b074592951e2e295a779e91 as hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/ns/7279affc7b074592951e2e295a779e91 2024-11-20T15:05:10,993 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/ns/7279affc7b074592951e2e295a779e91, entries=2, sequenceid=61, filesize=5.0 K 2024-11-20T15:05:10,995 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/table/490d71ae8c9d40a085bb108ea1f06219 as hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/table/490d71ae8c9d40a085bb108ea1f06219 2024-11-20T15:05:11,005 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/table/490d71ae8c9d40a085bb108ea1f06219, entries=2, sequenceid=61, filesize=5.0 K 2024-11-20T15:05:11,011 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 253ms, sequenceid=61, compaction requested=false 2024-11-20T15:05:11,012 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T15:05:11,024 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-20T15:05:11,027 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T15:05:11,027 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T15:05:11,027 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732115110753Running coprocessor pre-close hooks at 1732115110753Disabling compacts and flushes for region at 1732115110753Disabling writes for close at 1732115110753Obtaining lock to block concurrent updates at 1732115110759 (+6 ms)Preparing flush snapshotting stores in 1588230740 at 1732115110759Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732115110770 (+11 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732115110771 (+1 ms)Flushing 1588230740/info: creating writer at 1732115110773 (+2 ms)Flushing 1588230740/info: appending metadata at 1732115110829 (+56 ms)Flushing 1588230740/info: closing flushed file at 1732115110832 (+3 ms)Flushing 1588230740/ns: creating writer at 1732115110898 (+66 ms)Flushing 1588230740/ns: appending metadata at 1732115110914 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732115110914Flushing 1588230740/table: creating writer at 1732115110934 (+20 ms)Flushing 1588230740/table: appending metadata at 1732115110950 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732115110950Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39b50a2d: reopening flushed file at 1732115110969 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cfcdd1d: reopening flushed file at 1732115110982 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7df3f4c1: reopening flushed file at 1732115110993 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 253ms, sequenceid=61, compaction requested=false at 1732115111011 (+18 ms)Writing region close event to WAL at 1732115111018 (+7 ms)Running coprocessor post-close hooks at 1732115111025 (+7 ms)Closed at 1732115111027 (+2 ms) 2024-11-20T15:05:11,028 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] regionserver.HRegionServer(3302): Adding 1588230740 move to 1a15ecfd95f4,36179,1732115105395 record at close sequenceid=61 2024-11-20T15:05:11,031 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META, pid=58}] handler.UnassignRegionHandler(157): Closed 1588230740 2024-11-20T15:05:11,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=1588230740, regionState=CLOSED 2024-11-20T15:05:11,032 WARN [PEWorker-1 {}] zookeeper.MetaTableLocator(168): Tried to set null ServerName in hbase:meta; skipping -- ServerName required 2024-11-20T15:05:11,033 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=58, ppid=57, state=RUNNABLE, hasLock=true; CloseRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:11,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-20T15:05:11,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 in 439 msec 2024-11-20T15:05:11,037 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE; state=CLOSED, location=1a15ecfd95f4,36179,1732115105395; forceNewPlan=false, retain=false 2024-11-20T15:05:11,187 INFO [1a15ecfd95f4:39091 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T15:05:11,188 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:11,189 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,36179,1732115105395, state=OPENING 2024-11-20T15:05:11,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:11,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:11,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:11,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:11,259 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:11,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:11,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:11,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:11,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,36179,1732115105395}] 2024-11-20T15:05:11,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:11,421 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T15:05:11,421 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T15:05:11,421 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T15:05:11,424 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a15ecfd95f4%2C36179%2C1732115105395.meta, suffix=.meta, logDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395, archiveDir=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs, maxLogs=32 2024-11-20T15:05:11,443 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, exclude list is [], retry=0 2024-11-20T15:05:11,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34807,DS-6246c99a-c186-42f4-960c-91269a275f0d,DISK] 2024-11-20T15:05:11,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38935,DS-a0b7ff8b-cc8f-4112-bea6-13ca8f08a4a2,DISK] 2024-11-20T15:05:11,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42487,DS-77d67676-1e5f-4a3d-9c70-d4f0caedd6ad,DISK] 2024-11-20T15:05:11,448 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta 2024-11-20T15:05:11,449 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44829:44829),(127.0.0.1/127.0.0.1:42775:42775),(127.0.0.1/127.0.0.1:42085:42085)] 2024-11-20T15:05:11,449 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T15:05:11,449 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T15:05:11,450 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T15:05:11,450 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T15:05:11,450 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T15:05:11,450 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:11,450 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T15:05:11,450 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T15:05:11,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T15:05:11,454 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T15:05:11,454 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:11,464 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/info/3bfb7ea755e747edbf6d73a666ef2234 2024-11-20T15:05:11,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:11,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T15:05:11,466 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T15:05:11,466 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:11,475 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/ns/7279affc7b074592951e2e295a779e91 2024-11-20T15:05:11,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:11,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T15:05:11,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T15:05:11,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:11,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:11,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T15:05:11,478 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T15:05:11,478 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:11,488 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/table/490d71ae8c9d40a085bb108ea1f06219 2024-11-20T15:05:11,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:11,488 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T15:05:11,489 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740 2024-11-20T15:05:11,491 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740 2024-11-20T15:05:11,493 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T15:05:11,493 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T15:05:11,494 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T15:05:11,496 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T15:05:11,497 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58878019, jitterRate=-0.12264914810657501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T15:05:11,498 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T15:05:11,499 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732115111450Writing region info on filesystem at 1732115111450Initializing all the Stores at 1732115111451 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115111451Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115111452 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115111452Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115111452Cleaning up temporary data from old regions at 1732115111493 (+41 ms)Running coprocessor post-open hooks at 1732115111498 (+5 ms)Region opened successfully at 1732115111498 2024-11-20T15:05:11,500 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=59, masterSystemTime=1732115111412 2024-11-20T15:05:11,502 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T15:05:11,502 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=59}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T15:05:11,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:11,504 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,36179,1732115105395, state=OPEN 2024-11-20T15:05:11,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:11,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:11,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:11,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:11,525 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=59, ppid=57, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:11,526 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:11,526 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:11,526 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:11,526 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:11,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-20T15:05:11,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,36179,1732115105395 in 266 msec 2024-11-20T15:05:11,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE in 998 msec 2024-11-20T15:05:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-20T15:05:11,538 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T15:05:11,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-20T15:05:11,564 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=393, OpenFileDescriptor=661, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=10033 2024-11-20T15:05:11,567 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-20T15:05:11,568 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-20T15:05:11,568 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:11,569 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 1a15ecfd95f4,36179,1732115105395: Simulated kill ***** 2024-11-20T15:05:11,569 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T15:05:11,572 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T15:05:11,575 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T15:05:11,575 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T15:05:11,575 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T15:05:11,576 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 368165840 }, "NonHeapMemoryUsage": { "committed": 156762112, "init": 7667712, "max": -1, "used": 151932792 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1a15ecfd95f4" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1a15ecfd95f4" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "1a15ecfd95f4" } ] } 2024-11-20T15:05:11,579 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39091 {}] master.MasterRpcServices(700): 1a15ecfd95f4,36179,1732115105395 reported a fatal error: ***** ABORTING region server 1a15ecfd95f4,36179,1732115105395: Simulated kill ***** 2024-11-20T15:05:11,581 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a15ecfd95f4,36179,1732115105395' ***** 2024-11-20T15:05:11,582 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-20T15:05:11,582 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T15:05:11,582 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T15:05:11,582 INFO [RS:2;1a15ecfd95f4:36179 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T15:05:11,582 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T15:05:11,582 INFO [RS:2;1a15ecfd95f4:36179 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T15:05:11,582 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T15:05:11,583 INFO [RS:2;1a15ecfd95f4:36179 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1a15ecfd95f4:36179. 2024-11-20T15:05:11,583 DEBUG [RS:2;1a15ecfd95f4:36179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T15:05:11,587 DEBUG [RS:2;1a15ecfd95f4:36179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:11,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741867_1043 (size=436) 2024-11-20T15:05:11,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741867_1043 (size=436) 2024-11-20T15:05:11,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741867_1043 (size=436) 2024-11-20T15:05:11,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741835_1011 (size=2237) 2024-11-20T15:05:11,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741835_1011 (size=2237) 2024-11-20T15:05:11,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741835_1011 (size=2237) 2024-11-20T15:05:11,599 DEBUG [RS:2;1a15ecfd95f4:36179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:11,599 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T15:05:11,600 INFO [RS:2;1a15ecfd95f4:36179 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36179 2024-11-20T15:05:11,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a15ecfd95f4,36179,1732115105395 2024-11-20T15:05:11,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T15:05:11,617 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T15:05:11,626 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a15ecfd95f4,36179,1732115105395] 2024-11-20T15:05:11,634 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a15ecfd95f4,36179,1732115105395 already deleted, retry=false 2024-11-20T15:05:11,634 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1a15ecfd95f4,36179,1732115105395 on 1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:11,638 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:11,642 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 1a15ecfd95f4,36179,1732115105395 (carryingMeta=true) 1a15ecfd95f4,36179,1732115105395/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6cf164cf[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T15:05:11,643 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:11,645 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true, isMeta: true 2024-11-20T15:05:11,647 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting 2024-11-20T15:05:11,648 INFO [regionserver/1a15ecfd95f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T15:05:11,649 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1a15ecfd95f4,36179,1732115105395 WAL count=1, meta=true 2024-11-20T15:05:11,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta}] 2024-11-20T15:05:11,660 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:11,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:11,684 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T15:05:11,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T15:05:11,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1015a10928d0003, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T15:05:11,726 INFO [RS:2;1a15ecfd95f4:36179 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T15:05:11,727 INFO [RS:2;1a15ecfd95f4:36179 {}] regionserver.HRegionServer(1031): Exiting; stopping=1a15ecfd95f4,36179,1732115105395; zookeeper connection closed. 2024-11-20T15:05:11,727 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@518906b2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@518906b2 2024-11-20T15:05:11,815 INFO [regionserver/1a15ecfd95f4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T15:05:11,815 INFO [regionserver/1a15ecfd95f4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T15:05:11,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-20T15:05:11,853 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, size=436 (436bytes) 2024-11-20T15:05:11,854 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T15:05:11,854 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T15:05:11,854 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta 2024-11-20T15:05:11,859 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta after 3ms 2024-11-20T15:05:11,866 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta took 14ms 2024-11-20T15:05:11,875 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T15:05:11,875 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta so closing down 2024-11-20T15:05:11,876 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T15:05:11,876 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T15:05:11,879 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 1 edits across 0 Regions in 9 ms; skipped=1; WAL=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, size=436, length=436, corrupted=false, cancelled=false 2024-11-20T15:05:11,879 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, journal: Splitting hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, size=436 (436bytes) at 1732115111853Finishing writing output for hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta so closing down at 1732115111875 (+22 ms)3 split writer threads finished at 1732115111876 (+1 ms)Processed 1 edits across 0 Regions in 9 ms; skipped=1; WAL=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, size=436, length=436, corrupted=false, cancelled=false at 1732115111879 (+3 ms) 2024-11-20T15:05:11,880 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta 2024-11-20T15:05:11,881 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-20T15:05:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-20T15:05:11,889 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta to hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs 2024-11-20T15:05:11,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-20T15:05:11,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 in 228 msec 2024-11-20T15:05:11,894 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:11,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T15:05:11,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 in 243 msec 2024-11-20T15:05:11,899 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting dir is empty, no logs to split. 2024-11-20T15:05:11,899 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1a15ecfd95f4,36179,1732115105395 WAL count=0, meta=true 2024-11-20T15:05:11,899 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1a15ecfd95f4,36179,1732115105395 WAL splitting is done? wals=0, meta=true 2024-11-20T15:05:11,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T15:05:11,903 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:11,903 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,054 DEBUG [1a15ecfd95f4:39091 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T15:05:12,054 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T15:05:12,054 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T15:05:12,054 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T15:05:12,054 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T15:05:12,054 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T15:05:12,054 INFO [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T15:05:12,054 INFO [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T15:05:12,054 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T15:05:12,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,056 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPENING 2024-11-20T15:05:12,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:12,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:12,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:12,100 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:12,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:12,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:12,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:12,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:12,258 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T15:05:12,259 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T15:05:12,259 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T15:05:12,259 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T15:05:12,259 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T15:05:12,259 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T15:05:12,259 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,259 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T15:05:12,259 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T15:05:12,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T15:05:12,263 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T15:05:12,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,271 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/info/3bfb7ea755e747edbf6d73a666ef2234 2024-11-20T15:05:12,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:12,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T15:05:12,273 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T15:05:12,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,281 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/ns/7279affc7b074592951e2e295a779e91 2024-11-20T15:05:12,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:12,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T15:05:12,282 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T15:05:12,282 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:12,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T15:05:12,284 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T15:05:12,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,292 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/table/490d71ae8c9d40a085bb108ea1f06219 2024-11-20T15:05:12,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T15:05:12,292 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T15:05:12,293 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740 2024-11-20T15:05:12,294 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740 2024-11-20T15:05:12,296 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T15:05:12,296 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T15:05:12,297 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T15:05:12,298 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T15:05:12,299 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67176929, jitterRate=0.0010142475366592407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T15:05:12,300 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T15:05:12,300 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732115112260Writing region info on filesystem at 1732115112260Initializing all the Stores at 1732115112261 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115112261Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115112261Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112261Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732115112261Cleaning up temporary data from old regions at 1732115112296 (+35 ms)Running coprocessor post-open hooks at 1732115112300 (+4 ms)Region opened successfully at 1732115112300 2024-11-20T15:05:12,301 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732115112253 2024-11-20T15:05:12,303 DEBUG [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T15:05:12,303 INFO [RS_OPEN_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T15:05:12,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,305 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:12,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:12,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:12,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:12,334 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:12,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:12,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:12,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-20T15:05:12,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 in 233 msec 2024-11-20T15:05:12,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-20T15:05:12,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 436 msec 2024-11-20T15:05:12,341 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 1a15ecfd95f4,36179,1732115105395 had 9 regions 2024-11-20T15:05:12,343 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true, isMeta: false 2024-11-20T15:05:12,345 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1a15ecfd95f4,36179,1732115105395 WAL count=1, meta=false 2024-11-20T15:05:12,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914}] 2024-11-20T15:05:12,347 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:12,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-20T15:05:12,517 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, size=2.2 K (2237bytes) 2024-11-20T15:05:12,517 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914 2024-11-20T15:05:12,518 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914 after 1ms 2024-11-20T15:05:12,521 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914 took 4ms 2024-11-20T15:05:12,524 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for edc47ba2cc61e109ae6ff5bc44000143: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T15:05:12,526 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 26f2170e12c973cd322aaede01781706: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T15:05:12,528 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 69575a66848849b3da0176c19f735873: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T15:05:12,530 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 82aa8da18f9efb24d07c39d806b4c510: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T15:05:12,532 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b955aba2c76f221f15758639fba8cacb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T15:05:12,534 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4bcef47d7fdf6bb3fb9febb5d5b742ea: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T15:05:12,535 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 67e17e52ef4d4eb06c5ffca586b82a9f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T15:05:12,537 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 92fc6039f588e69ef1ddcd4fd69369db: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T15:05:12,537 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914 so closing down 2024-11-20T15:05:12,537 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T15:05:12,537 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T15:05:12,537 INFO [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-20T15:05:12,537 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, journal: Splitting hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, size=2.2 K (2237bytes) at 1732115112517Finishing writing output for hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914 so closing down at 1732115112537 (+20 ms)3 split writer threads finished at 1732115112537Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732115112537 2024-11-20T15:05:12,537 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914 2024-11-20T15:05:12,537 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1a15ecfd95f4:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-20T15:05:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39091 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-20T15:05:12,542 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting/1a15ecfd95f4%2C36179%2C1732115105395.1732115106914 to hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs 2024-11-20T15:05:12,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-20T15:05:12,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 in 194 msec 2024-11-20T15:05:12,546 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-20T15:05:12,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 in 201 msec 2024-11-20T15:05:12,551 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/WALs/1a15ecfd95f4,36179,1732115105395-splitting dir is empty, no logs to split. 2024-11-20T15:05:12,551 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1a15ecfd95f4,36179,1732115105395 WAL count=0, meta=false 2024-11-20T15:05:12,551 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1a15ecfd95f4,36179,1732115105395 WAL splitting is done? wals=0, meta=false 2024-11-20T15:05:12,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN}] 2024-11-20T15:05:12,556 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:12,557 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,558 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:12,558 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,559 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:12,560 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,561 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:12,562 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,562 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:12,563 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,564 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:12,565 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,565 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:12,566 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,567 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:12,568 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T15:05:12,707 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(204): Hosts are {1a15ecfd95f4=0} racks are {/default-rack=0} 2024-11-20T15:05:12,707 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T15:05:12,708 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T15:05:12,708 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T15:05:12,708 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T15:05:12,708 INFO [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T15:05:12,708 INFO [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T15:05:12,708 DEBUG [1a15ecfd95f4:39091 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T15:05:12,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, regionLocation=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:12,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=26f2170e12c973cd322aaede01781706, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:12,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN because future has completed 2024-11-20T15:05:12,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:12,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN because future has completed 2024-11-20T15:05:12,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN because future has completed 2024-11-20T15:05:12,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN because future has completed 2024-11-20T15:05:12,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN because future has completed 2024-11-20T15:05:12,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:12,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN because future has completed 2024-11-20T15:05:12,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:12,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:12,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:12,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345}] 2024-11-20T15:05:12,732 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=69575a66848849b3da0176c19f735873, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:12,732 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, regionLocation=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:12,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN because future has completed 2024-11-20T15:05:12,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:12,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN because future has completed 2024-11-20T15:05:12,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67e17e52ef4d4eb06c5ffca586b82a9f, server=1a15ecfd95f4,42753,1732115105221}] 2024-11-20T15:05:12,869 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:12,870 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 4bcef47d7fdf6bb3fb9febb5d5b742ea, NAME => 'test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-20T15:05:12,870 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,870 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,870 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,870 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,872 INFO [StoreOpener-4bcef47d7fdf6bb3fb9febb5d5b742ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,873 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:12,873 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 67e17e52ef4d4eb06c5ffca586b82a9f, NAME => 'test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-20T15:05:12,873 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,873 INFO [StoreOpener-4bcef47d7fdf6bb3fb9febb5d5b742ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4bcef47d7fdf6bb3fb9febb5d5b742ea columnFamilyName family 2024-11-20T15:05:12,873 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,873 DEBUG [StoreOpener-4bcef47d7fdf6bb3fb9febb5d5b742ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,873 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,873 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,874 INFO [StoreOpener-4bcef47d7fdf6bb3fb9febb5d5b742ea-1 {}] regionserver.HStore(327): Store=4bcef47d7fdf6bb3fb9febb5d5b742ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:12,874 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,875 INFO [StoreOpener-67e17e52ef4d4eb06c5ffca586b82a9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,875 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,876 INFO [StoreOpener-67e17e52ef4d4eb06c5ffca586b82a9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67e17e52ef4d4eb06c5ffca586b82a9f columnFamilyName family 2024-11-20T15:05:12,876 DEBUG [StoreOpener-67e17e52ef4d4eb06c5ffca586b82a9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,877 INFO [StoreOpener-67e17e52ef4d4eb06c5ffca586b82a9f-1 {}] regionserver.HStore(327): Store=67e17e52ef4d4eb06c5ffca586b82a9f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:12,877 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,877 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,877 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,877 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,878 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,879 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,879 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,879 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,879 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,880 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 4bcef47d7fdf6bb3fb9febb5d5b742ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69704416, jitterRate=0.03867673873901367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:12,880 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:12,880 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 4bcef47d7fdf6bb3fb9febb5d5b742ea: Running coprocessor pre-open hook at 1732115112871Writing region info on filesystem at 1732115112871Initializing all the Stores at 1732115112871Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112872 (+1 ms)Cleaning up temporary data from old regions at 1732115112877 (+5 ms)Running coprocessor post-open hooks at 1732115112880 (+3 ms)Region opened successfully at 1732115112880 2024-11-20T15:05:12,881 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea., pid=80, masterSystemTime=1732115112865 2024-11-20T15:05:12,881 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,882 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 67e17e52ef4d4eb06c5ffca586b82a9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72115477, jitterRate=0.0746043473482132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:12,882 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:12,883 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 67e17e52ef4d4eb06c5ffca586b82a9f: Running coprocessor pre-open hook at 1732115112874Writing region info on filesystem at 1732115112874Initializing all the Stores at 1732115112875 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112875Cleaning up temporary data from old regions at 1732115112879 (+4 ms)Running coprocessor post-open hooks at 1732115112882 (+3 ms)Region opened successfully at 1732115112883 (+1 ms) 2024-11-20T15:05:12,883 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f., pid=82, masterSystemTime=1732115112868 2024-11-20T15:05:12,884 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:12,884 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:12,884 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:12,884 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 92fc6039f588e69ef1ddcd4fd69369db, NAME => 'test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-20T15:05:12,885 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,885 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,885 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,885 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,886 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:12,886 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:12,886 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:12,886 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 82aa8da18f9efb24d07c39d806b4c510, NAME => 'test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-20T15:05:12,886 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-20T15:05:12,886 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,886 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,887 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:12,887 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,887 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,887 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:12,891 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T15:05:12,892 INFO [StoreOpener-92fc6039f588e69ef1ddcd4fd69369db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,892 INFO [StoreOpener-82aa8da18f9efb24d07c39d806b4c510-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,893 INFO [StoreOpener-92fc6039f588e69ef1ddcd4fd69369db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92fc6039f588e69ef1ddcd4fd69369db columnFamilyName family 2024-11-20T15:05:12,894 INFO [StoreOpener-82aa8da18f9efb24d07c39d806b4c510-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82aa8da18f9efb24d07c39d806b4c510 columnFamilyName family 2024-11-20T15:05:12,894 DEBUG [StoreOpener-92fc6039f588e69ef1ddcd4fd69369db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,894 DEBUG [StoreOpener-82aa8da18f9efb24d07c39d806b4c510-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,894 INFO [StoreOpener-82aa8da18f9efb24d07c39d806b4c510-1 {}] regionserver.HStore(327): Store=82aa8da18f9efb24d07c39d806b4c510/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:12,894 INFO [StoreOpener-92fc6039f588e69ef1ddcd4fd69369db-1 {}] regionserver.HStore(327): Store=92fc6039f588e69ef1ddcd4fd69369db/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:12,894 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,895 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,895 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,896 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,897 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,898 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,898 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,898 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,898 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,898 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,898 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-20T15:05:12,901 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,901 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:12,902 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67e17e52ef4d4eb06c5ffca586b82a9f, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:12,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:12,902 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67e17e52ef4d4eb06c5ffca586b82a9f, server=1a15ecfd95f4,42753,1732115105221 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:12,903 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 82aa8da18f9efb24d07c39d806b4c510; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60214274, jitterRate=-0.10273739695549011}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:12,903 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:12,903 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 92fc6039f588e69ef1ddcd4fd69369db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67022462, jitterRate=-0.0012874901294708252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:12,903 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:12,903 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 82aa8da18f9efb24d07c39d806b4c510: Running coprocessor pre-open hook at 1732115112887Writing region info on filesystem at 1732115112887Initializing all the Stores at 1732115112888 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112888Cleaning up temporary data from old regions at 1732115112898 (+10 ms)Running coprocessor post-open hooks at 1732115112903 (+5 ms)Region opened successfully at 1732115112903 2024-11-20T15:05:12,903 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 92fc6039f588e69ef1ddcd4fd69369db: Running coprocessor pre-open hook at 1732115112885Writing region info on filesystem at 1732115112885Initializing all the Stores at 1732115112886 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112887 (+1 ms)Cleaning up temporary data from old regions at 1732115112898 (+11 ms)Running coprocessor post-open hooks at 1732115112903 (+5 ms)Region opened successfully at 1732115112903 2024-11-20T15:05:12,904 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db., pid=78, masterSystemTime=1732115112865 2024-11-20T15:05:12,904 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510., pid=76, masterSystemTime=1732115112868 2024-11-20T15:05:12,907 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:12,907 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:12,907 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:12,907 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => edc47ba2cc61e109ae6ff5bc44000143, NAME => 'test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-20T15:05:12,908 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,908 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,908 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,908 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,908 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:12,908 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:12,908 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:12,909 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 26f2170e12c973cd322aaede01781706, NAME => 'test,rrr,1732115107710.26f2170e12c973cd322aaede01781706.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-20T15:05:12,909 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,909 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,rrr,1732115107710.26f2170e12c973cd322aaede01781706.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,909 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,909 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,909 INFO [StoreOpener-edc47ba2cc61e109ae6ff5bc44000143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,911 INFO [StoreOpener-edc47ba2cc61e109ae6ff5bc44000143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edc47ba2cc61e109ae6ff5bc44000143 columnFamilyName family 2024-11-20T15:05:12,911 DEBUG [StoreOpener-edc47ba2cc61e109ae6ff5bc44000143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,911 INFO [StoreOpener-26f2170e12c973cd322aaede01781706-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,911 INFO [StoreOpener-edc47ba2cc61e109ae6ff5bc44000143-1 {}] regionserver.HStore(327): Store=edc47ba2cc61e109ae6ff5bc44000143/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:12,911 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,912 INFO [StoreOpener-26f2170e12c973cd322aaede01781706-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26f2170e12c973cd322aaede01781706 columnFamilyName family 2024-11-20T15:05:12,912 DEBUG [StoreOpener-26f2170e12c973cd322aaede01781706-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,912 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,912 INFO [StoreOpener-26f2170e12c973cd322aaede01781706-1 {}] regionserver.HStore(327): Store=26f2170e12c973cd322aaede01781706/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:12,913 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,913 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,913 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,914 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,914 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,914 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,915 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,915 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,916 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,917 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened edc47ba2cc61e109ae6ff5bc44000143; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74291850, jitterRate=0.107034832239151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:12,917 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,917 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:12,917 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for edc47ba2cc61e109ae6ff5bc44000143: Running coprocessor pre-open hook at 1732115112908Writing region info on filesystem at 1732115112908Initializing all the Stores at 1732115112909 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112909Cleaning up temporary data from old regions at 1732115112914 (+5 ms)Running coprocessor post-open hooks at 1732115112917 (+3 ms)Region opened successfully at 1732115112917 2024-11-20T15:05:12,918 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143., pid=79, masterSystemTime=1732115112865 2024-11-20T15:05:12,918 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 26f2170e12c973cd322aaede01781706; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68743483, jitterRate=0.024357721209526062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:12,918 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:12,918 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 26f2170e12c973cd322aaede01781706: Running coprocessor pre-open hook at 1732115112910Writing region info on filesystem at 1732115112910Initializing all the Stores at 1732115112910Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112910Cleaning up temporary data from old regions at 1732115112915 (+5 ms)Running coprocessor post-open hooks at 1732115112918 (+3 ms)Region opened successfully at 1732115112918 2024-11-20T15:05:12,919 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732115107710.26f2170e12c973cd322aaede01781706., pid=77, masterSystemTime=1732115112868 2024-11-20T15:05:12,921 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:12,921 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:12,921 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:12,921 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => b955aba2c76f221f15758639fba8cacb, NAME => 'test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-20T15:05:12,922 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,922 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,922 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,922 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,923 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:12,923 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:12,923 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:12,923 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 69575a66848849b3da0176c19f735873, NAME => 'test,lll,1732115107710.69575a66848849b3da0176c19f735873.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-20T15:05:12,923 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,924 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,lll,1732115107710.69575a66848849b3da0176c19f735873.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T15:05:12,924 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,924 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,924 INFO [StoreOpener-b955aba2c76f221f15758639fba8cacb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,925 INFO [StoreOpener-69575a66848849b3da0176c19f735873-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,926 INFO [StoreOpener-b955aba2c76f221f15758639fba8cacb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b955aba2c76f221f15758639fba8cacb columnFamilyName family 2024-11-20T15:05:12,926 DEBUG [StoreOpener-b955aba2c76f221f15758639fba8cacb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,927 INFO [StoreOpener-b955aba2c76f221f15758639fba8cacb-1 {}] regionserver.HStore(327): Store=b955aba2c76f221f15758639fba8cacb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:12,927 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,927 INFO [StoreOpener-69575a66848849b3da0176c19f735873-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69575a66848849b3da0176c19f735873 columnFamilyName family 2024-11-20T15:05:12,927 DEBUG [StoreOpener-69575a66848849b3da0176c19f735873-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T15:05:12,928 INFO [StoreOpener-69575a66848849b3da0176c19f735873-1 {}] regionserver.HStore(327): Store=69575a66848849b3da0176c19f735873/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T15:05:12,928 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,928 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,929 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,930 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,930 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,930 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,930 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,931 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,931 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,933 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,933 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,934 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 69575a66848849b3da0176c19f735873; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74802922, jitterRate=0.11465039849281311}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:12,934 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened b955aba2c76f221f15758639fba8cacb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62340758, jitterRate=-0.07105031609535217}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T15:05:12,934 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:12,934 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:12,934 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for b955aba2c76f221f15758639fba8cacb: Running coprocessor pre-open hook at 1732115112922Writing region info on filesystem at 1732115112922Initializing all the Stores at 1732115112924 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112924Cleaning up temporary data from old regions at 1732115112930 (+6 ms)Running coprocessor post-open hooks at 1732115112934 (+4 ms)Region opened successfully at 1732115112934 2024-11-20T15:05:12,934 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 69575a66848849b3da0176c19f735873: Running coprocessor pre-open hook at 1732115112924Writing region info on filesystem at 1732115112924Initializing all the Stores at 1732115112925 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732115112925Cleaning up temporary data from old regions at 1732115112931 (+6 ms)Running coprocessor post-open hooks at 1732115112934 (+3 ms)Region opened successfully at 1732115112934 2024-11-20T15:05:12,935 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb., pid=75, masterSystemTime=1732115112865 2024-11-20T15:05:12,935 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732115107710.69575a66848849b3da0176c19f735873., pid=81, masterSystemTime=1732115112868 2024-11-20T15:05:12,937 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:12,937 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:12,938 DEBUG [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:12,939 INFO [RS_OPEN_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:12,956 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-20T15:05:12,987 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:12,987 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:12,987 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:12,987 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:12,988 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:12,989 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:12,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:12,989 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:12,990 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,002 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,005 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,005 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,007 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67e17e52ef4d4eb06c5ffca586b82a9f, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,012 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,014 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,014 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,014 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67e17e52ef4d4eb06c5ffca586b82a9f, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,014 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 23 msec 2024-11-20T15:05:13,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,016 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,017 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,085 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,085 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,085 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,085 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,085 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,085 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,085 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,085 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,086 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143086 2024-11-20T15:05:13,086 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,086 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,088 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,089 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,42753,1732115105221 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,116 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,116 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,116 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173116 2024-11-20T15:05:13,116 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233116 2024-11-20T15:05:13,117 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-20T15:05:13,117 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,117 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,117 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,117 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,117 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,117 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,117 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,117 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,118 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,122 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,123 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,127 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,128 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,128 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,129 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,129 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,129 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,129 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69575a66848849b3da0176c19f735873, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-20T15:05:13,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,130 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,130 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,143 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,143 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,143 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,143 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,143 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,143 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,143 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,143 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,144 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143144 2024-11-20T15:05:13,144 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,144 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,146 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,147 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,148 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,158 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,158 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173158 2024-11-20T15:05:13,158 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233158 2024-11-20T15:05:13,158 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-20T15:05:13,158 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,159 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,159 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,159 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,159 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,159 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,159 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,159 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,170 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-20T15:05:13,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,171 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,172 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,185 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143186 2024-11-20T15:05:13,187 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,186 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,191 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,201 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,201 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173201 2024-11-20T15:05:13,201 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233201 2024-11-20T15:05:13,201 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-20T15:05:13,201 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,201 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,201 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,201 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,201 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,202 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,202 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure edc47ba2cc61e109ae6ff5bc44000143, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,212 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T15:05:13,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,213 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,214 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,264 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,265 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143265 2024-11-20T15:05:13,265 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,265 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,266 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,267 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,278 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,278 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173278 2024-11-20T15:05:13,279 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233278 2024-11-20T15:05:13,279 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-20T15:05:13,279 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,279 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,279 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,279 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,279 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,280 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,280 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,288 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,288 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,290 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,290 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,290 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92fc6039f588e69ef1ddcd4fd69369db, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,291 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-20T15:05:13,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,292 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,293 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,300 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,300 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,300 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,302 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143302 2024-11-20T15:05:13,302 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,302 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,303 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,304 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,304 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,305 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,313 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173313 2024-11-20T15:05:13,314 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233313 2024-11-20T15:05:13,314 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-20T15:05:13,314 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,314 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,314 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,314 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,314 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,314 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,314 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,315 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,315 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,318 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,318 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,318 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,319 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,324 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,324 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,324 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26f2170e12c973cd322aaede01781706, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,325 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T15:05:13,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,327 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,327 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,335 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,335 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143335 2024-11-20T15:05:13,335 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,335 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,336 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,337 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,337 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,337 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,345 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173345 2024-11-20T15:05:13,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233345 2024-11-20T15:05:13,345 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-20T15:05:13,345 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,345 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,345 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,345 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,345 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,346 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,346 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,346 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,350 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,350 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82aa8da18f9efb24d07c39d806b4c510, server=1a15ecfd95f4,42753,1732115105221 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,353 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T15:05:13,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,354 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,354 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,359 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,359 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,360 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,360 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143360 2024-11-20T15:05:13,360 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,361 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,362 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,364 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,364 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,364 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,370 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,370 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173370 2024-11-20T15:05:13,370 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233370 2024-11-20T15:05:13,370 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T15:05:13,370 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,370 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,370 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,371 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,371 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,371 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,371 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,371 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,372 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,375 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,375 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,375 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,375 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,378 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure b955aba2c76f221f15758639fba8cacb, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,379 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T15:05:13,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,380 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,380 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,391 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,392 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,393 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,395 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143395 2024-11-20T15:05:13,395 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,395 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,396 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,397 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,397 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,398 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,407 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,407 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173407 2024-11-20T15:05:13,407 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233407 2024-11-20T15:05:13,407 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-20T15:05:13,407 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,408 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,408 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,408 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,408 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,408 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,408 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,409 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,414 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,414 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN 2024-11-20T15:05:13,418 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-20T15:05:13,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,420 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,420 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,426 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,426 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,426 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,426 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,426 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,426 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,426 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,426 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea, ASSIGN to state=OFFLINE, location=null, table=test, region=4bcef47d7fdf6bb3fb9febb5d5b742ea to restore RIT 2024-11-20T15:05:13,427 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143427 2024-11-20T15:05:13,428 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,428 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,429 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,430 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,430 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,430 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,438 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,438 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173438 2024-11-20T15:05:13,438 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233438 2024-11-20T15:05:13,438 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-20T15:05:13,438 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,438 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,439 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,439 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,439 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,439 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,439 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,439 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,440 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,443 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,444 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,447 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,447 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,447 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,447 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,447 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,448 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,448 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,448 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,448 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,448 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN 2024-11-20T15:05:13,448 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T15:05:13,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,449 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,449 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,459 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,459 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,459 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,460 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,460 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,460 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,460 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,460 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,460 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,460 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edc47ba2cc61e109ae6ff5bc44000143, ASSIGN to state=OFFLINE, location=null, table=test, region=edc47ba2cc61e109ae6ff5bc44000143 to restore RIT 2024-11-20T15:05:13,461 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143461 2024-11-20T15:05:13,461 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,461 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,463 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,464 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,464 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,464 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,472 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,472 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173472 2024-11-20T15:05:13,472 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233472 2024-11-20T15:05:13,472 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-20T15:05:13,473 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,473 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,473 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,473 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,473 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,473 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,473 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,474 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,475 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,482 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,482 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,483 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,483 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,483 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN 2024-11-20T15:05:13,483 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-20T15:05:13,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,484 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,484 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,492 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,492 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,493 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,493 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,493 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,493 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b955aba2c76f221f15758639fba8cacb, ASSIGN to state=OFFLINE, location=null, table=test, region=b955aba2c76f221f15758639fba8cacb to restore RIT 2024-11-20T15:05:13,493 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143493 2024-11-20T15:05:13,493 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,493 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,495 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,499 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,499 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,499 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,503 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,503 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173503 2024-11-20T15:05:13,503 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233503 2024-11-20T15:05:13,503 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-20T15:05:13,503 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,503 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,503 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,503 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,504 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,504 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,504 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,505 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,511 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,512 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,512 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN 2024-11-20T15:05:13,513 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T15:05:13,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,514 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,514 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,548 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,548 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,548 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,548 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,548 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26f2170e12c973cd322aaede01781706, ASSIGN to state=OFFLINE, location=null, table=test, region=26f2170e12c973cd322aaede01781706 to restore RIT 2024-11-20T15:05:13,549 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143548 2024-11-20T15:05:13,549 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,549 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,550 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,551 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,551 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,551 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,556 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,556 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173556 2024-11-20T15:05:13,556 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233556 2024-11-20T15:05:13,556 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T15:05:13,556 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,556 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,556 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,556 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,556 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,556 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,556 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,556 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,561 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,561 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN 2024-11-20T15:05:13,561 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T15:05:13,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,562 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,562 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,576 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=92fc6039f588e69ef1ddcd4fd69369db, ASSIGN to state=OFFLINE, location=null, table=test, region=92fc6039f588e69ef1ddcd4fd69369db to restore RIT 2024-11-20T15:05:13,576 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143576 2024-11-20T15:05:13,576 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,576 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,578 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,579 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,579 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,579 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,584 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,584 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173584 2024-11-20T15:05:13,584 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233584 2024-11-20T15:05:13,584 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T15:05:13,584 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,584 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,584 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,584 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,585 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,585 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,585 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,585 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,586 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,587 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,588 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,588 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,588 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,589 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,590 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,590 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,590 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,590 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,590 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,590 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,590 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN 2024-11-20T15:05:13,590 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T15:05:13,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,591 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,591 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,598 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,598 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,601 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,601 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,601 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=69575a66848849b3da0176c19f735873, ASSIGN to state=OFFLINE, location=null, table=test, region=69575a66848849b3da0176c19f735873 to restore RIT 2024-11-20T15:05:13,602 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143601 2024-11-20T15:05:13,602 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,602 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,603 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,603 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,603 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,604 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,609 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,609 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173609 2024-11-20T15:05:13,609 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233609 2024-11-20T15:05:13,610 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T15:05:13,610 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,610 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,610 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,610 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,610 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,611 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,613 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,617 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN 2024-11-20T15:05:13,617 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T15:05:13,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,618 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,618 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=82aa8da18f9efb24d07c39d806b4c510, ASSIGN to state=OFFLINE, location=null, table=test, region=82aa8da18f9efb24d07c39d806b4c510 to restore RIT 2024-11-20T15:05:13,626 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143626 2024-11-20T15:05:13,626 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,626 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,628 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,629 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,629 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,629 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,633 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,633 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173633 2024-11-20T15:05:13,633 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233633 2024-11-20T15:05:13,633 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T15:05:13,633 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,633 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,633 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,633 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,633 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,633 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,633 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,634 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,634 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,636 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,638 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,638 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN 2024-11-20T15:05:13,639 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T15:05:13,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,639 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,639 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,651 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f, ASSIGN to state=OFFLINE, location=null, table=test, region=67e17e52ef4d4eb06c5ffca586b82a9f to restore RIT 2024-11-20T15:05:13,651 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143651 2024-11-20T15:05:13,651 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,651 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,653 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,654 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,654 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,654 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,660 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,660 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173660 2024-11-20T15:05:13,660 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233660 2024-11-20T15:05:13,660 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-20T15:05:13,660 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,660 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,660 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,661 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,661 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,661 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,662 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,664 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,666 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T15:05:13,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,668 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,668 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,676 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143676 2024-11-20T15:05:13,676 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,676 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,678 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,679 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,679 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,679 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,683 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,683 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173683 2024-11-20T15:05:13,684 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233683 2024-11-20T15:05:13,684 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T15:05:13,684 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,684 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,684 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,684 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,684 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,684 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,684 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,684 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,685 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.1732115106914, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T15:05:13,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,689 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,689 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,701 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143701 2024-11-20T15:05:13,701 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,701 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,702 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,706 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,706 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,706 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,709 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,709 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173709 2024-11-20T15:05:13,709 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233709 2024-11-20T15:05:13,709 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T15:05:13,709 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,709 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,709 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,709 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,710 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,710 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,710 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,710 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T15:05:13,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,714 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T15:05:13,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,715 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,715 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,726 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143726 2024-11-20T15:05:13,726 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,726 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,727 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,728 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,728 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,729 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,732 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,732 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173732 2024-11-20T15:05:13,732 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233732 2024-11-20T15:05:13,732 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T15:05:13,732 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,732 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,733 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,733 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,733 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,733 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,733 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,734 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T15:05:13,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,736 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,737 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,737 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T15:05:13,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,737 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,738 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,742 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,742 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,742 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,742 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143742 2024-11-20T15:05:13,742 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,742 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,744 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,747 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,747 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,748 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,748 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173748 2024-11-20T15:05:13,748 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233748 2024-11-20T15:05:13,749 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T15:05:13,749 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,749 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,749 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,749 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,749 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,749 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,749 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,749 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,751 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,751 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,751 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T15:05:13,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,752 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,752 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,752 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T15:05:13,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,753 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,753 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,759 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143759 2024-11-20T15:05:13,760 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,760 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,761 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,763 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,767 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173767 2024-11-20T15:05:13,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233767 2024-11-20T15:05:13,767 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-20T15:05:13,767 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,767 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,767 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,767 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,767 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,767 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,769 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,769 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T15:05:13,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,770 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1a15ecfd95f4%2C36179%2C1732115105395.meta.1732115111426.meta, worker=1a15ecfd95f4,39199,1732115105345 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T15:05:13,770 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T15:05:13,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,771 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,771 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,784 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143784 2024-11-20T15:05:13,784 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,784 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,785 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,786 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T15:05:13,786 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,786 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,790 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,790 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173790 2024-11-20T15:05:13,790 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233790 2024-11-20T15:05:13,790 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T15:05:13,791 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T15:05:13,791 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,791 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:13,791 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:13,791 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T15:05:13,791 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T15:05:13,791 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T15:05:13,791 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T15:05:13,792 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T15:05:13,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T15:05:13,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T15:05:13,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T15:05:13,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, REOPEN/MOVE 2024-11-20T15:05:13,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true 2024-11-20T15:05:13,795 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T15:05:13,795 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-20T15:05:13,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=65 2024-11-20T15:05:13,796 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1a15ecfd95f4,39199,1732115105345, table=hbase:meta, region=1588230740 2024-11-20T15:05:13,796 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a15ecfd95f4,39199,1732115105345, state=OPEN 2024-11-20T15:05:13,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T15:05:13,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T15:05:13,801 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732115143801 2024-11-20T15:05:13,801 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T15:05:13,801 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T15:05:13,802 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T15:05:13,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=74, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4bcef47d7fdf6bb3fb9febb5d5b742ea, server=1a15ecfd95f4,39199,1732115105345:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1a15ecfd95f4,36179,1732115105395, splitWal=true, meta=true exec-time=2.1670 sec 2024-11-20T15:05:13,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dda0575ccaa2bf831399fa5e62de1046, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f62894fb325f2b3b65c5f08886f870, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92fc6039f588e69ef1ddcd4fd69369db, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50775deb03089a468fbaec014df1a140, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67e17e52ef4d4eb06c5ffca586b82a9f, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=804e74fa77c72dca2dc16c98cdc6f0a1, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3185afc44c3f735f4f17ff966a0953af, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37573db7e6f70ad7de5e799e1f1056e6, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edc47ba2cc61e109ae6ff5bc44000143, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fff7a80be69d0e9493705d50119cbd2c, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201c3d6de103d545e918088a64fbb1d, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69575a66848849b3da0176c19f735873, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=29a8b1d98f980f11f2986d2cd2aca2ff, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd4f4852f3df2562a02cbbf2795ee89, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4bcef47d7fdf6bb3fb9febb5d5b742ea, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7315d7c27f8d82e7df1af9ed20d4c351, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1142c4fcbc5da26a9d7a32d826bc9c00, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26f2170e12c973cd322aaede01781706, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=600616f3e3ce04088a974ba2eb27ffcc, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=730ee1d6e6dcb8ce4e182db74bb5e284, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b955aba2c76f221f15758639fba8cacb, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12dda3ed00bfb1c1662ec93d7e6e2e11, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96b1c73fb3ef1f0a71731c98fed31de8, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8816e1f68bf2af371e2eeff20fb184ae, regionState=OPEN, lastHost=1a15ecfd95f4,42753,1732115105221, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40fff0ed2a6c7cab41e9aeca535ada94, regionState=OPEN, lastHost=1a15ecfd95f4,39199,1732115105345, regionLocation=1a15ecfd95f4,39199,1732115105345, openSeqNum=2 2024-11-20T15:05:13,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82aa8da18f9efb24d07c39d806b4c510, regionState=OPENING, lastHost=1a15ecfd95f4,36179,1732115105395, regionLocation=1a15ecfd95f4,42753,1732115105221, openSeqNum=2 2024-11-20T15:05:13,807 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T15:05:13,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732115173807 2024-11-20T15:05:13,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732115233807 2024-11-20T15:05:13,807 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-20T15:05:13,822 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=351 (was 393), OpenFileDescriptor=626 (was 661), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 127) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10010 (was 10033) 2024-11-20T15:05:13,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T15:05:13,822 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T15:05:13,822 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T15:05:13,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:13,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:13,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T15:05:13,823 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T15:05:13,823 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=780736434, stopped=false 2024-11-20T15:05:13,823 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1a15ecfd95f4,39091,1732115104505 2024-11-20T15:05:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T15:05:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T15:05:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T15:05:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:13,833 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:13,834 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T15:05:13,834 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T15:05:13,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:13,834 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a15ecfd95f4,42753,1732115105221' ***** 2024-11-20T15:05:13,834 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T15:05:13,834 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1a15ecfd95f4,39199,1732115105345' ***** 2024-11-20T15:05:13,834 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T15:05:13,834 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T15:05:13,834 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T15:05:13,835 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T15:05:13,835 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T15:05:13,835 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T15:05:13,835 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T15:05:13,835 INFO [RS:0;1a15ecfd95f4:42753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T15:05:13,835 INFO [RS:1;1a15ecfd95f4:39199 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T15:05:13,835 INFO [RS:0;1a15ecfd95f4:42753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T15:05:13,835 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T15:05:13,835 INFO [RS:1;1a15ecfd95f4:39199 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T15:05:13,835 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 4bcef47d7fdf6bb3fb9febb5d5b742ea 2024-11-20T15:05:13,835 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:13,836 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:13,836 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for b955aba2c76f221f15758639fba8cacb 2024-11-20T15:05:13,836 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 600616f3e3ce04088a974ba2eb27ffcc 2024-11-20T15:05:13,836 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 804e74fa77c72dca2dc16c98cdc6f0a1 2024-11-20T15:05:13,836 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 12dda3ed00bfb1c1662ec93d7e6e2e11 2024-11-20T15:05:13,836 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 730ee1d6e6dcb8ce4e182db74bb5e284 2024-11-20T15:05:13,836 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 1142c4fcbc5da26a9d7a32d826bc9c00 2024-11-20T15:05:13,836 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 92fc6039f588e69ef1ddcd4fd69369db 2024-11-20T15:05:13,836 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 8816e1f68bf2af371e2eeff20fb184ae 2024-11-20T15:05:13,836 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for dda0575ccaa2bf831399fa5e62de1046 2024-11-20T15:05:13,836 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 3185afc44c3f735f4f17ff966a0953af 2024-11-20T15:05:13,836 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 2201c3d6de103d545e918088a64fbb1d 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 26f2170e12c973cd322aaede01781706 2024-11-20T15:05:13,837 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 40fff0ed2a6c7cab41e9aeca535ada94 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 69575a66848849b3da0176c19f735873 2024-11-20T15:05:13,837 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 7315d7c27f8d82e7df1af9ed20d4c351 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 67e17e52ef4d4eb06c5ffca586b82a9f 2024-11-20T15:05:13,837 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 37573db7e6f70ad7de5e799e1f1056e6 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 50775deb03089a468fbaec014df1a140 2024-11-20T15:05:13,837 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 0fd4f4852f3df2562a02cbbf2795ee89 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 29a8b1d98f980f11f2986d2cd2aca2ff 2024-11-20T15:05:13,837 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 96b1c73fb3ef1f0a71731c98fed31de8 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 65f62894fb325f2b3b65c5f08886f870 2024-11-20T15:05:13,837 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4bcef47d7fdf6bb3fb9febb5d5b742ea, disabling compactions & flushes 2024-11-20T15:05:13,837 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fff7a80be69d0e9493705d50119cbd2c, disabling compactions & flushes 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(3091): Received CLOSE for 82aa8da18f9efb24d07c39d806b4c510 2024-11-20T15:05:13,837 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(959): stopping server 1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(959): stopping server 1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:13,837 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:13,837 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:13,837 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T15:05:13,837 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:13,837 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T15:05:13,837 INFO [RS:1;1a15ecfd95f4:39199 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1a15ecfd95f4:39199. 2024-11-20T15:05:13,837 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. after waiting 0 ms 2024-11-20T15:05:13,837 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. after waiting 0 ms 2024-11-20T15:05:13,837 INFO [RS:0;1a15ecfd95f4:42753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1a15ecfd95f4:42753. 2024-11-20T15:05:13,837 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:13,837 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:13,837 DEBUG [RS:1;1a15ecfd95f4:39199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T15:05:13,837 DEBUG [RS:0;1a15ecfd95f4:42753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T15:05:13,838 DEBUG [RS:1;1a15ecfd95f4:39199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:13,838 DEBUG [RS:0;1a15ecfd95f4:42753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:13,838 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T15:05:13,838 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T15:05:13,838 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-20T15:05:13,838 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T15:05:13,838 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T15:05:13,838 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-20T15:05:13,838 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T15:05:13,839 DEBUG [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(1351): Waiting on 1142c4fcbc5da26a9d7a32d826bc9c00, 12dda3ed00bfb1c1662ec93d7e6e2e11, 26f2170e12c973cd322aaede01781706, 29a8b1d98f980f11f2986d2cd2aca2ff, 3185afc44c3f735f4f17ff966a0953af, 50775deb03089a468fbaec014df1a140, 600616f3e3ce04088a974ba2eb27ffcc, 65f62894fb325f2b3b65c5f08886f870, 67e17e52ef4d4eb06c5ffca586b82a9f, 69575a66848849b3da0176c19f735873, 82aa8da18f9efb24d07c39d806b4c510, 8816e1f68bf2af371e2eeff20fb184ae, fff7a80be69d0e9493705d50119cbd2c 2024-11-20T15:05:13,839 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T15:05:13,839 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T15:05:13,839 DEBUG [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(1351): Waiting on 0fd4f4852f3df2562a02cbbf2795ee89, 1588230740, 2201c3d6de103d545e918088a64fbb1d, 37573db7e6f70ad7de5e799e1f1056e6, 40fff0ed2a6c7cab41e9aeca535ada94, 4bcef47d7fdf6bb3fb9febb5d5b742ea, 730ee1d6e6dcb8ce4e182db74bb5e284, 7315d7c27f8d82e7df1af9ed20d4c351, 804e74fa77c72dca2dc16c98cdc6f0a1, 92fc6039f588e69ef1ddcd4fd69369db, 96b1c73fb3ef1f0a71731c98fed31de8, b955aba2c76f221f15758639fba8cacb, dda0575ccaa2bf831399fa5e62de1046, edc47ba2cc61e109ae6ff5bc44000143 2024-11-20T15:05:13,839 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T15:05:13,839 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T15:05:13,839 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-11-20T15:05:13,844 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/fff7a80be69d0e9493705d50119cbd2c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,845 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/4bcef47d7fdf6bb3fb9febb5d5b742ea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,845 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:13,845 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fff7a80be69d0e9493705d50119cbd2c: Waiting for close lock at 1732115113836Running coprocessor pre-close hooks at 1732115113837 (+1 ms)Disabling compacts and flushes for region at 1732115113837Disabling writes for close at 1732115113837Writing region close event to WAL at 1732115113839 (+2 ms)Running coprocessor post-close hooks at 1732115113845 (+6 ms)Closed at 1732115113845 2024-11-20T15:05:13,845 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732115107710.fff7a80be69d0e9493705d50119cbd2c. 2024-11-20T15:05:13,845 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 600616f3e3ce04088a974ba2eb27ffcc, disabling compactions & flushes 2024-11-20T15:05:13,845 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:13,845 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:13,845 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. after waiting 0 ms 2024-11-20T15:05:13,845 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:13,846 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:13,846 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4bcef47d7fdf6bb3fb9febb5d5b742ea: Waiting for close lock at 1732115113836Running coprocessor pre-close hooks at 1732115113837 (+1 ms)Disabling compacts and flushes for region at 1732115113837Disabling writes for close at 1732115113837Writing region close event to WAL at 1732115113839 (+2 ms)Running coprocessor post-close hooks at 1732115113845 (+6 ms)Closed at 1732115113846 (+1 ms) 2024-11-20T15:05:13,846 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732115107710.4bcef47d7fdf6bb3fb9febb5d5b742ea. 2024-11-20T15:05:13,846 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing edc47ba2cc61e109ae6ff5bc44000143, disabling compactions & flushes 2024-11-20T15:05:13,846 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:13,846 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:13,846 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. after waiting 0 ms 2024-11-20T15:05:13,846 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:13,850 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/600616f3e3ce04088a974ba2eb27ffcc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,850 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/edc47ba2cc61e109ae6ff5bc44000143/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,851 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:13,851 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 600616f3e3ce04088a974ba2eb27ffcc: Waiting for close lock at 1732115113845Running coprocessor pre-close hooks at 1732115113845Disabling compacts and flushes for region at 1732115113845Disabling writes for close at 1732115113845Writing region close event to WAL at 1732115113846 (+1 ms)Running coprocessor post-close hooks at 1732115113850 (+4 ms)Closed at 1732115113851 (+1 ms) 2024-11-20T15:05:13,851 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732115107710.600616f3e3ce04088a974ba2eb27ffcc. 2024-11-20T15:05:13,851 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12dda3ed00bfb1c1662ec93d7e6e2e11, disabling compactions & flushes 2024-11-20T15:05:13,851 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:13,851 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:13,851 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. after waiting 0 ms 2024-11-20T15:05:13,851 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:13,851 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:13,851 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for edc47ba2cc61e109ae6ff5bc44000143: Waiting for close lock at 1732115113846Running coprocessor pre-close hooks at 1732115113846Disabling compacts and flushes for region at 1732115113846Disabling writes for close at 1732115113846Writing region close event to WAL at 1732115113847 (+1 ms)Running coprocessor post-close hooks at 1732115113851 (+4 ms)Closed at 1732115113851 2024-11-20T15:05:13,852 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732115107710.edc47ba2cc61e109ae6ff5bc44000143. 2024-11-20T15:05:13,852 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b955aba2c76f221f15758639fba8cacb, disabling compactions & flushes 2024-11-20T15:05:13,852 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:13,852 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:13,852 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. after waiting 0 ms 2024-11-20T15:05:13,852 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:13,855 INFO [regionserver/1a15ecfd95f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T15:05:13,855 INFO [regionserver/1a15ecfd95f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T15:05:13,858 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/b955aba2c76f221f15758639fba8cacb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,859 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:13,859 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b955aba2c76f221f15758639fba8cacb: Waiting for close lock at 1732115113852Running coprocessor pre-close hooks at 1732115113852Disabling compacts and flushes for region at 1732115113852Disabling writes for close at 1732115113852Writing region close event to WAL at 1732115113854 (+2 ms)Running coprocessor post-close hooks at 1732115113859 (+5 ms)Closed at 1732115113859 2024-11-20T15:05:13,859 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732115107710.b955aba2c76f221f15758639fba8cacb. 2024-11-20T15:05:13,859 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 804e74fa77c72dca2dc16c98cdc6f0a1, disabling compactions & flushes 2024-11-20T15:05:13,859 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:13,859 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:13,859 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. after waiting 0 ms 2024-11-20T15:05:13,860 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:13,863 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/info/17dec942309948a591bf764d16c0bbd3 is 138, key is test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db./info:regioninfo/1732115112710/Put/seqid=0 2024-11-20T15:05:13,864 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/12dda3ed00bfb1c1662ec93d7e6e2e11/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,865 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:13,865 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12dda3ed00bfb1c1662ec93d7e6e2e11: Waiting for close lock at 1732115113851Running coprocessor pre-close hooks at 1732115113851Disabling compacts and flushes for region at 1732115113851Disabling writes for close at 1732115113851Writing region close event to WAL at 1732115113853 (+2 ms)Running coprocessor post-close hooks at 1732115113865 (+12 ms)Closed at 1732115113865 2024-11-20T15:05:13,865 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732115107710.12dda3ed00bfb1c1662ec93d7e6e2e11. 2024-11-20T15:05:13,866 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1142c4fcbc5da26a9d7a32d826bc9c00, disabling compactions & flushes 2024-11-20T15:05:13,866 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:13,866 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:13,866 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. after waiting 0 ms 2024-11-20T15:05:13,866 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:13,867 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/804e74fa77c72dca2dc16c98cdc6f0a1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,868 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:13,868 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 804e74fa77c72dca2dc16c98cdc6f0a1: Waiting for close lock at 1732115113859Running coprocessor pre-close hooks at 1732115113859Disabling compacts and flushes for region at 1732115113859Disabling writes for close at 1732115113859Writing region close event to WAL at 1732115113860 (+1 ms)Running coprocessor post-close hooks at 1732115113868 (+8 ms)Closed at 1732115113868 2024-11-20T15:05:13,868 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732115107710.804e74fa77c72dca2dc16c98cdc6f0a1. 2024-11-20T15:05:13,868 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 730ee1d6e6dcb8ce4e182db74bb5e284, disabling compactions & flushes 2024-11-20T15:05:13,869 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:13,869 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:13,869 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. after waiting 0 ms 2024-11-20T15:05:13,869 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:13,881 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/730ee1d6e6dcb8ce4e182db74bb5e284/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,881 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/1142c4fcbc5da26a9d7a32d826bc9c00/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741868_1044 (size=8192) 2024-11-20T15:05:13,882 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:13,882 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1142c4fcbc5da26a9d7a32d826bc9c00: Waiting for close lock at 1732115113866Running coprocessor pre-close hooks at 1732115113866Disabling compacts and flushes for region at 1732115113866Disabling writes for close at 1732115113866Writing region close event to WAL at 1732115113873 (+7 ms)Running coprocessor post-close hooks at 1732115113882 (+9 ms)Closed at 1732115113882 2024-11-20T15:05:13,882 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732115107710.1142c4fcbc5da26a9d7a32d826bc9c00. 2024-11-20T15:05:13,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741868_1044 (size=8192) 2024-11-20T15:05:13,882 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8816e1f68bf2af371e2eeff20fb184ae, disabling compactions & flushes 2024-11-20T15:05:13,883 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:13,883 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 730ee1d6e6dcb8ce4e182db74bb5e284: Waiting for close lock at 1732115113868Running coprocessor pre-close hooks at 1732115113868Disabling compacts and flushes for region at 1732115113868Disabling writes for close at 1732115113869 (+1 ms)Writing region close event to WAL at 1732115113874 (+5 ms)Running coprocessor post-close hooks at 1732115113882 (+8 ms)Closed at 1732115113882 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. after waiting 0 ms 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732115107710.730ee1d6e6dcb8ce4e182db74bb5e284. 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 92fc6039f588e69ef1ddcd4fd69369db, disabling compactions & flushes 2024-11-20T15:05:13,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741868_1044 (size=8192) 2024-11-20T15:05:13,883 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. after waiting 0 ms 2024-11-20T15:05:13,883 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:13,883 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/info/17dec942309948a591bf764d16c0bbd3 2024-11-20T15:05:13,888 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/92fc6039f588e69ef1ddcd4fd69369db/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,889 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:13,889 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/8816e1f68bf2af371e2eeff20fb184ae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,890 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 92fc6039f588e69ef1ddcd4fd69369db: Waiting for close lock at 1732115113883Running coprocessor pre-close hooks at 1732115113883Disabling compacts and flushes for region at 1732115113883Disabling writes for close at 1732115113883Writing region close event to WAL at 1732115113884 (+1 ms)Running coprocessor post-close hooks at 1732115113889 (+5 ms)Closed at 1732115113889 2024-11-20T15:05:13,890 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732115107710.92fc6039f588e69ef1ddcd4fd69369db. 2024-11-20T15:05:13,890 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dda0575ccaa2bf831399fa5e62de1046, disabling compactions & flushes 2024-11-20T15:05:13,890 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:13,890 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:13,890 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. after waiting 0 ms 2024-11-20T15:05:13,890 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:13,890 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:13,890 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8816e1f68bf2af371e2eeff20fb184ae: Waiting for close lock at 1732115113882Running coprocessor pre-close hooks at 1732115113882Disabling compacts and flushes for region at 1732115113882Disabling writes for close at 1732115113883 (+1 ms)Writing region close event to WAL at 1732115113884 (+1 ms)Running coprocessor post-close hooks at 1732115113890 (+6 ms)Closed at 1732115113890 2024-11-20T15:05:13,890 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732115107710.8816e1f68bf2af371e2eeff20fb184ae. 2024-11-20T15:05:13,891 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3185afc44c3f735f4f17ff966a0953af, disabling compactions & flushes 2024-11-20T15:05:13,891 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:13,891 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:13,891 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. after waiting 0 ms 2024-11-20T15:05:13,891 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:13,894 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/.tmp/info/17dec942309948a591bf764d16c0bbd3 as hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/info/17dec942309948a591bf764d16c0bbd3 2024-11-20T15:05:13,895 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/dda0575ccaa2bf831399fa5e62de1046/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,895 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/3185afc44c3f735f4f17ff966a0953af/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,896 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:13,896 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:13,896 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dda0575ccaa2bf831399fa5e62de1046: Waiting for close lock at 1732115113890Running coprocessor pre-close hooks at 1732115113890Disabling compacts and flushes for region at 1732115113890Disabling writes for close at 1732115113890Writing region close event to WAL at 1732115113891 (+1 ms)Running coprocessor post-close hooks at 1732115113896 (+5 ms)Closed at 1732115113896 2024-11-20T15:05:13,896 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3185afc44c3f735f4f17ff966a0953af: Waiting for close lock at 1732115113891Running coprocessor pre-close hooks at 1732115113891Disabling compacts and flushes for region at 1732115113891Disabling writes for close at 1732115113891Writing region close event to WAL at 1732115113891Running coprocessor post-close hooks at 1732115113896 (+5 ms)Closed at 1732115113896 2024-11-20T15:05:13,896 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732115107710.dda0575ccaa2bf831399fa5e62de1046. 2024-11-20T15:05:13,896 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732115107710.3185afc44c3f735f4f17ff966a0953af. 2024-11-20T15:05:13,897 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2201c3d6de103d545e918088a64fbb1d, disabling compactions & flushes 2024-11-20T15:05:13,897 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 26f2170e12c973cd322aaede01781706, disabling compactions & flushes 2024-11-20T15:05:13,897 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:13,897 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:13,897 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:13,897 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:13,897 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. after waiting 0 ms 2024-11-20T15:05:13,897 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. after waiting 0 ms 2024-11-20T15:05:13,897 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:13,897 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:13,902 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/26f2170e12c973cd322aaede01781706/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,903 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/2201c3d6de103d545e918088a64fbb1d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,903 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:13,903 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 26f2170e12c973cd322aaede01781706: Waiting for close lock at 1732115113897Running coprocessor pre-close hooks at 1732115113897Disabling compacts and flushes for region at 1732115113897Disabling writes for close at 1732115113897Writing region close event to WAL at 1732115113898 (+1 ms)Running coprocessor post-close hooks at 1732115113903 (+5 ms)Closed at 1732115113903 2024-11-20T15:05:13,903 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732115107710.26f2170e12c973cd322aaede01781706. 2024-11-20T15:05:13,903 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:13,903 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 69575a66848849b3da0176c19f735873, disabling compactions & flushes 2024-11-20T15:05:13,903 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2201c3d6de103d545e918088a64fbb1d: Waiting for close lock at 1732115113897Running coprocessor pre-close hooks at 1732115113897Disabling compacts and flushes for region at 1732115113897Disabling writes for close at 1732115113897Writing region close event to WAL at 1732115113897Running coprocessor post-close hooks at 1732115113903 (+6 ms)Closed at 1732115113903 2024-11-20T15:05:13,903 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:13,903 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:13,903 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732115107710.2201c3d6de103d545e918088a64fbb1d. 2024-11-20T15:05:13,903 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732115107710.69575a66848849b3da0176c19f735873. after waiting 0 ms 2024-11-20T15:05:13,904 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:13,904 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40fff0ed2a6c7cab41e9aeca535ada94, disabling compactions & flushes 2024-11-20T15:05:13,904 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/info/17dec942309948a591bf764d16c0bbd3, entries=24, sequenceid=75, filesize=8 K 2024-11-20T15:05:13,904 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:13,904 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:13,904 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. after waiting 0 ms 2024-11-20T15:05:13,904 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:13,905 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 66ms, sequenceid=75, compaction requested=false 2024-11-20T15:05:13,908 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/69575a66848849b3da0176c19f735873/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,908 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/40fff0ed2a6c7cab41e9aeca535ada94/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,909 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:13,909 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40fff0ed2a6c7cab41e9aeca535ada94: Waiting for close lock at 1732115113904Running coprocessor pre-close hooks at 1732115113904Disabling compacts and flushes for region at 1732115113904Disabling writes for close at 1732115113904Writing region close event to WAL at 1732115113904Running coprocessor post-close hooks at 1732115113909 (+5 ms)Closed at 1732115113909 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 69575a66848849b3da0176c19f735873: Waiting for close lock at 1732115113903Running coprocessor pre-close hooks at 1732115113903Disabling compacts and flushes for region at 1732115113903Disabling writes for close at 1732115113903Writing region close event to WAL at 1732115113904 (+1 ms)Running coprocessor post-close hooks at 1732115113909 (+5 ms)Closed at 1732115113909 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732115107710.40fff0ed2a6c7cab41e9aeca535ada94. 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732115107710.69575a66848849b3da0176c19f735873. 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7315d7c27f8d82e7df1af9ed20d4c351, disabling compactions & flushes 2024-11-20T15:05:13,909 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 67e17e52ef4d4eb06c5ffca586b82a9f, disabling compactions & flushes 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:13,909 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. after waiting 0 ms 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. after waiting 0 ms 2024-11-20T15:05:13,909 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:13,913 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-20T15:05:13,914 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T15:05:13,914 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/7315d7c27f8d82e7df1af9ed20d4c351/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,914 INFO [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T15:05:13,914 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732115113838Running coprocessor pre-close hooks at 1732115113838Disabling compacts and flushes for region at 1732115113838Disabling writes for close at 1732115113839 (+1 ms)Obtaining lock to block concurrent updates at 1732115113839Preparing flush snapshotting stores in 1588230740 at 1732115113839Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732115113840 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732115113841 (+1 ms)Flushing 1588230740/info: creating writer at 1732115113841Flushing 1588230740/info: appending metadata at 1732115113863 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732115113863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4914a33: reopening flushed file at 1732115113893 (+30 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 66ms, sequenceid=75, compaction requested=false at 1732115113905 (+12 ms)Writing region close event to WAL at 1732115113907 (+2 ms)Running coprocessor post-close hooks at 1732115113913 (+6 ms)Closed at 1732115113914 (+1 ms) 2024-11-20T15:05:13,914 DEBUG [RS_CLOSE_META-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T15:05:13,914 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:13,915 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7315d7c27f8d82e7df1af9ed20d4c351: Waiting for close lock at 1732115113909Running coprocessor pre-close hooks at 1732115113909Disabling compacts and flushes for region at 1732115113909Disabling writes for close at 1732115113909Writing region close event to WAL at 1732115113910 (+1 ms)Running coprocessor post-close hooks at 1732115113914 (+4 ms)Closed at 1732115113914 2024-11-20T15:05:13,915 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732115107710.7315d7c27f8d82e7df1af9ed20d4c351. 2024-11-20T15:05:13,915 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 37573db7e6f70ad7de5e799e1f1056e6, disabling compactions & flushes 2024-11-20T15:05:13,915 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:13,915 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:13,915 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. after waiting 0 ms 2024-11-20T15:05:13,915 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:13,915 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/67e17e52ef4d4eb06c5ffca586b82a9f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,916 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:13,916 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 67e17e52ef4d4eb06c5ffca586b82a9f: Waiting for close lock at 1732115113909Running coprocessor pre-close hooks at 1732115113909Disabling compacts and flushes for region at 1732115113909Disabling writes for close at 1732115113909Writing region close event to WAL at 1732115113910 (+1 ms)Running coprocessor post-close hooks at 1732115113916 (+6 ms)Closed at 1732115113916 2024-11-20T15:05:13,916 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732115107710.67e17e52ef4d4eb06c5ffca586b82a9f. 2024-11-20T15:05:13,916 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 50775deb03089a468fbaec014df1a140, disabling compactions & flushes 2024-11-20T15:05:13,916 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:13,917 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:13,917 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. after waiting 0 ms 2024-11-20T15:05:13,917 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:13,921 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/50775deb03089a468fbaec014df1a140/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,921 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/37573db7e6f70ad7de5e799e1f1056e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,922 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:13,922 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 50775deb03089a468fbaec014df1a140: Waiting for close lock at 1732115113916Running coprocessor pre-close hooks at 1732115113916Disabling compacts and flushes for region at 1732115113916Disabling writes for close at 1732115113917 (+1 ms)Writing region close event to WAL at 1732115113917Running coprocessor post-close hooks at 1732115113922 (+5 ms)Closed at 1732115113922 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 37573db7e6f70ad7de5e799e1f1056e6: Waiting for close lock at 1732115113915Running coprocessor pre-close hooks at 1732115113915Disabling compacts and flushes for region at 1732115113915Disabling writes for close at 1732115113915Writing region close event to WAL at 1732115113915Running coprocessor post-close hooks at 1732115113922 (+7 ms)Closed at 1732115113922 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732115107710.50775deb03089a468fbaec014df1a140. 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732115107710.37573db7e6f70ad7de5e799e1f1056e6. 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 29a8b1d98f980f11f2986d2cd2aca2ff, disabling compactions & flushes 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0fd4f4852f3df2562a02cbbf2795ee89, disabling compactions & flushes 2024-11-20T15:05:13,922 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:13,922 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. after waiting 0 ms 2024-11-20T15:05:13,922 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. after waiting 0 ms 2024-11-20T15:05:13,923 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:13,923 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:13,927 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/29a8b1d98f980f11f2986d2cd2aca2ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,927 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/0fd4f4852f3df2562a02cbbf2795ee89/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,928 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:13,928 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 29a8b1d98f980f11f2986d2cd2aca2ff: Waiting for close lock at 1732115113922Running coprocessor pre-close hooks at 1732115113922Disabling compacts and flushes for region at 1732115113922Disabling writes for close at 1732115113922Writing region close event to WAL at 1732115113923 (+1 ms)Running coprocessor post-close hooks at 1732115113928 (+5 ms)Closed at 1732115113928 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0fd4f4852f3df2562a02cbbf2795ee89: Waiting for close lock at 1732115113922Running coprocessor pre-close hooks at 1732115113922Disabling compacts and flushes for region at 1732115113922Disabling writes for close at 1732115113922Writing region close event to WAL at 1732115113923 (+1 ms)Running coprocessor post-close hooks at 1732115113928 (+5 ms)Closed at 1732115113928 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732115107710.29a8b1d98f980f11f2986d2cd2aca2ff. 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732115107710.0fd4f4852f3df2562a02cbbf2795ee89. 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 96b1c73fb3ef1f0a71731c98fed31de8, disabling compactions & flushes 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 65f62894fb325f2b3b65c5f08886f870, disabling compactions & flushes 2024-11-20T15:05:13,928 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:13,928 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. after waiting 0 ms 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. after waiting 0 ms 2024-11-20T15:05:13,928 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:13,934 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/96b1c73fb3ef1f0a71731c98fed31de8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,935 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:13,935 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 96b1c73fb3ef1f0a71731c98fed31de8: Waiting for close lock at 1732115113928Running coprocessor pre-close hooks at 1732115113928Disabling compacts and flushes for region at 1732115113928Disabling writes for close at 1732115113928Writing region close event to WAL at 1732115113930 (+2 ms)Running coprocessor post-close hooks at 1732115113934 (+4 ms)Closed at 1732115113935 (+1 ms) 2024-11-20T15:05:13,935 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732115107710.96b1c73fb3ef1f0a71731c98fed31de8. 2024-11-20T15:05:13,935 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/65f62894fb325f2b3b65c5f08886f870/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,935 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:13,935 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 65f62894fb325f2b3b65c5f08886f870: Waiting for close lock at 1732115113928Running coprocessor pre-close hooks at 1732115113928Disabling compacts and flushes for region at 1732115113928Disabling writes for close at 1732115113928Writing region close event to WAL at 1732115113930 (+2 ms)Running coprocessor post-close hooks at 1732115113935 (+5 ms)Closed at 1732115113935 2024-11-20T15:05:13,936 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732115107710.65f62894fb325f2b3b65c5f08886f870. 2024-11-20T15:05:13,936 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 82aa8da18f9efb24d07c39d806b4c510, disabling compactions & flushes 2024-11-20T15:05:13,936 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:13,936 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:13,936 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. after waiting 0 ms 2024-11-20T15:05:13,936 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:13,940 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/data/default/test/82aa8da18f9efb24d07c39d806b4c510/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T15:05:13,940 INFO [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:13,940 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 82aa8da18f9efb24d07c39d806b4c510: Waiting for close lock at 1732115113936Running coprocessor pre-close hooks at 1732115113936Disabling compacts and flushes for region at 1732115113936Disabling writes for close at 1732115113936Writing region close event to WAL at 1732115113936Running coprocessor post-close hooks at 1732115113940 (+4 ms)Closed at 1732115113940 2024-11-20T15:05:13,940 DEBUG [RS_CLOSE_REGION-regionserver/1a15ecfd95f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732115107710.82aa8da18f9efb24d07c39d806b4c510. 2024-11-20T15:05:14,039 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.HRegionServer(976): stopping server 1a15ecfd95f4,42753,1732115105221; all regions closed. 2024-11-20T15:05:14,039 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.HRegionServer(976): stopping server 1a15ecfd95f4,39199,1732115105345; all regions closed. 2024-11-20T15:05:14,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741834_1010 (size=7074) 2024-11-20T15:05:14,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741836_1012 (size=36954) 2024-11-20T15:05:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741836_1012 (size=36954) 2024-11-20T15:05:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741834_1010 (size=7074) 2024-11-20T15:05:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741836_1012 (size=36954) 2024-11-20T15:05:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741834_1010 (size=7074) 2024-11-20T15:05:14,050 DEBUG [RS:1;1a15ecfd95f4:39199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs 2024-11-20T15:05:14,051 DEBUG [RS:0;1a15ecfd95f4:42753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs 2024-11-20T15:05:14,051 INFO [RS:1;1a15ecfd95f4:39199 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C39199%2C1732115105345.meta:.meta(num 1732115107344) 2024-11-20T15:05:14,051 INFO [RS:0;1a15ecfd95f4:42753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C42753%2C1732115105221:(num 1732115106914) 2024-11-20T15:05:14,051 DEBUG [RS:0;1a15ecfd95f4:42753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:14,051 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T15:05:14,051 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T15:05:14,051 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.ChoreService(370): Chore service for: regionserver/1a15ecfd95f4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T15:05:14,052 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T15:05:14,052 INFO [regionserver/1a15ecfd95f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T15:05:14,052 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T15:05:14,052 INFO [RS:0;1a15ecfd95f4:42753 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T15:05:14,052 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T15:05:14,052 INFO [RS:0;1a15ecfd95f4:42753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42753 2024-11-20T15:05:14,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741833_1009 (size=7064) 2024-11-20T15:05:14,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741833_1009 (size=7064) 2024-11-20T15:05:14,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741833_1009 (size=7064) 2024-11-20T15:05:14,056 DEBUG [RS:1;1a15ecfd95f4:39199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/oldWALs 2024-11-20T15:05:14,056 INFO [RS:1;1a15ecfd95f4:39199 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1a15ecfd95f4%2C39199%2C1732115105345:(num 1732115106914) 2024-11-20T15:05:14,056 DEBUG [RS:1;1a15ecfd95f4:39199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T15:05:14,056 INFO [RS:1;1a15ecfd95f4:39199 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T15:05:14,056 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T15:05:14,056 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.ChoreService(370): Chore service for: regionserver/1a15ecfd95f4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T15:05:14,056 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T15:05:14,056 INFO [regionserver/1a15ecfd95f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T15:05:14,057 INFO [RS:1;1a15ecfd95f4:39199 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39199 2024-11-20T15:05:14,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T15:05:14,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42753-0x1015a10928d0001, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a15ecfd95f4,42753,1732115105221 2024-11-20T15:05:14,067 INFO [RS:0;1a15ecfd95f4:42753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T15:05:14,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39199-0x1015a10928d0002, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a15ecfd95f4,39199,1732115105345 2024-11-20T15:05:14,075 INFO [RS:1;1a15ecfd95f4:39199 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T15:05:14,075 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a15ecfd95f4,42753,1732115105221] 2024-11-20T15:05:14,092 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a15ecfd95f4,42753,1732115105221 already deleted, retry=false 2024-11-20T15:05:14,092 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1a15ecfd95f4,42753,1732115105221 expired; onlineServers=1 2024-11-20T15:05:14,092 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a15ecfd95f4,39199,1732115105345] 2024-11-20T15:05:14,100 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1a15ecfd95f4,39199,1732115105345 already deleted, retry=false 2024-11-20T15:05:14,100 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1a15ecfd95f4,39199,1732115105345 expired; onlineServers=0 2024-11-20T15:05:14,100 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1a15ecfd95f4,39091,1732115104505' ***** 2024-11-20T15:05:14,100 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T15:05:14,101 INFO [M:0;1a15ecfd95f4:39091 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T15:05:14,101 INFO [M:0;1a15ecfd95f4:39091 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T15:05:14,101 DEBUG [M:0;1a15ecfd95f4:39091 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T15:05:14,101 DEBUG [M:0;1a15ecfd95f4:39091 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T15:05:14,101 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T15:05:14,101 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732115106485 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.small.0-1732115106485,5,FailOnTimeoutGroup] 2024-11-20T15:05:14,101 DEBUG [master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732115106484 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a15ecfd95f4:0:becomeActiveMaster-HFileCleaner.large.0-1732115106484,5,FailOnTimeoutGroup] 2024-11-20T15:05:14,102 INFO [M:0;1a15ecfd95f4:39091 {}] hbase.ChoreService(370): Chore service for: master/1a15ecfd95f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T15:05:14,102 INFO [M:0;1a15ecfd95f4:39091 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T15:05:14,102 DEBUG [M:0;1a15ecfd95f4:39091 {}] master.HMaster(1795): Stopping service threads 2024-11-20T15:05:14,102 INFO [M:0;1a15ecfd95f4:39091 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T15:05:14,103 INFO [M:0;1a15ecfd95f4:39091 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T15:05:14,103 INFO [M:0;1a15ecfd95f4:39091 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T15:05:14,103 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T15:05:14,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T15:05:14,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T15:05:14,109 DEBUG [M:0;1a15ecfd95f4:39091 {}] zookeeper.ZKUtil(347): master:39091-0x1015a10928d0000, quorum=127.0.0.1:56692, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T15:05:14,109 WARN [M:0;1a15ecfd95f4:39091 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T15:05:14,110 INFO [M:0;1a15ecfd95f4:39091 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/.lastflushedseqids 2024-11-20T15:05:14,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741869_1045 (size=1868) 2024-11-20T15:05:14,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741869_1045 (size=1868) 2024-11-20T15:05:14,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741869_1045 (size=1868) 2024-11-20T15:05:14,129 INFO [M:0;1a15ecfd95f4:39091 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T15:05:14,129 INFO [M:0;1a15ecfd95f4:39091 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T15:05:14,129 DEBUG [M:0;1a15ecfd95f4:39091 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T15:05:14,130 INFO [M:0;1a15ecfd95f4:39091 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T15:05:14,130 DEBUG [M:0;1a15ecfd95f4:39091 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T15:05:14,130 DEBUG [M:0;1a15ecfd95f4:39091 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T15:05:14,130 DEBUG [M:0;1a15ecfd95f4:39091 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T15:05:14,130 INFO [M:0;1a15ecfd95f4:39091 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.90 KB heapSize=316.77 KB 2024-11-20T15:05:14,148 DEBUG [M:0;1a15ecfd95f4:39091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/de4d7bf601e74481af9c0df3837d0247 is 82, key is hbase:meta,,1/info:regioninfo/1732115112304/Put/seqid=0 2024-11-20T15:05:14,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34807 is added to blk_1073741870_1046 (size=6342) 2024-11-20T15:05:14,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42487 is added to blk_1073741870_1046 (size=6342) 2024-11-20T15:05:14,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38935 is added to blk_1073741870_1046 (size=6342) 2024-11-20T15:05:14,156 INFO [M:0;1a15ecfd95f4:39091 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.79 KB at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/de4d7bf601e74481af9c0df3837d0247 2024-11-20T15:05:14,180 DEBUG [M:0;1a15ecfd95f4:39091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32953/user/jenkins/test-data/58b41d32-b85f-0e15-2b89-dd35be54fb3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7f29fc02271545578162cd9f78144672 is 2987, key is \x00\x00\x00\x00\x00\x00\x00